'use strict';

const fs = require('fs');
const url = require('url');
const {chromium, firefox} = require('playwright');
const TurndownService = require('turndown');
const got = require('../utils/got');
const cheerio = require('cheerio');
const chinese = require('chinese-simple2traditional');
const dayjs = require('dayjs');
const axios = require('axios');
const https = require('https');

const turndownService = new TurndownService();

const Service = require('egg').Service;

class CrawlerSvc extends Service {

  async lengdanxiong() {
    const param = {
      url: 'https://www.zhihu.com/people/12345-88-22-52/zvideos',
      author_info: {
        author_id: '64ce4658b1493990d38848dc',
        author_name: '冷淡熊',
        author_avatar: 'https://i2.hdslb.com/bfs/face/afe334d673fda9bd834923aa0a67f207120e5eee.jpg@240w_240h_1c_1s_!web-avatar-space-header.webp'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['搞笑']
      },
      selector_info: {
        box: '.List-item',
        title: '.ContentItem-title a',
        content: '.ContentItem-title a',
        pic: '.RichContent-cover-inner img',
      },
    }

    let topics = await this.ctx.service.commonSvc.commonFetch(param);
    topics.forEach(v => {
      v.is_video = true;
      v.pic = v.pic.replace('_200x0.jpg', '.jpg');
    });

    return topics;
  }

  async worldcupSina() {
    let totalTopics = [];

    const url = 'https://sports.sina.cn/qatar2022/index.d.html?vt=4';

    const tab = 'worldcup';
    const author_id = '6219cd66c65f291a70c30037';
    const author_name = '新浪新闻';
    const author_avatar = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/sina.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.feed-item-box');
      const eles = await page.$$('.feed-item-box');
      // await this.ctx.helper.autoScrollByLimit(page, 1000);

      for (let i = 0; i < eles.length; i++) {
        const el = eles[i];
        let title = await el.$eval('.cm_feedTit', v => v.innerText);
        console.log(i, title);
        let content = await el.$eval('div', v => v.getAttribute('data-golink'));
        let pic;
        if (await el.$('img')) {
          pic = await el.$eval('img', v => v.src);
        }

        let is_video = false;
        let tags = ['世界杯'];
        if (await el.$('.cm_btn_paly')) {
          is_video = true;
        }

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          is_video,
          tags,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error('新浪新闻世界杯爬取异常!!!');
      } else {
        this.logger.info('新浪新闻世界杯爬取成功, 数量为:', totalTopics.length);
      }
      return totalTopics;
    }

  }


  async worldcupPengpai() {
    const param = {
      url: 'https://www.thepaper.cn/channel_123034',
      author_info: {
        author_id: '6219ca83af06e718ebbcbbd8',
        author_name: '澎湃新闻',
        author_avatar: '/public/images/pengpai.png'
      },
      topic_info: {
        tab: 'worldcup',
        boards: ['世界杯']
      },
      selector_info: {
        box: '.ant-row > div',
        title: 'h2',
        content: 'a',
        pic: 'img',
      },
    }

    let topics = await this.ctx.service.commonSvc.commonLight(param);
    return topics;

  }


  async worldcupQQ() {
    const param = {
      url: 'https://new.qq.com/ch/worldcup/',
      author_info: {
        author_id: '62ece0288e1f347a5c98bf7b',
        author_name: '腾讯新闻',
        author_avatar: 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/tengxunxinwen.jpg'
      },
      topic_info: {
        tab: 'worldcup',
        boards: ['世界杯']
      },
      selector_info: {
        box: '.channel_mod > .list > li',
        title: 'h3 > a',
        content: 'h3 > a',
        pic: 'img',
        tags: '.tags > .tag'
      }
    }

    let topics = await this.ctx.service.commonSvc.commonFetch(param);
    return topics;

  }


  async worldcupSohu() {
    const param = {
      url: 'https://mp.sohu.com/profile?xpt=cHBhZzQ0NzIzNDQwNjhhM0Bzb2h1LmNvbQ==',
      author_info: {
        author_id: '62317f42b479c2424a0a0a19',
        author_name: '搜狐新闻',
        author_avatar: 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/sohu.png'
      },
      click: 'div.tap-rank-tap > div.outtap > div > div:nth-child(2) > div',
      topic_info: {
        tab: 'worldcup',
        boards: ['世界杯']
      },
      selector_info: {
        box: '.recommend-content-wrap > div',
        title: '.item-text-content-title,.title',
        content: 'a',
        intro: '.item-text-content-description',
        pic: 'img'
      }
    }

    let topics = await this.ctx.service.commonSvc.commonFetch(param);
    return topics;

  }

  async worldcupCctv() {
    const param = {
      url: 'https://worldcup.cctv.com/2022/news/',
      author_info: {
        author_id: '62ecaa6d600655615738f8f5',
        author_name: '央视新闻',
        author_avatar: 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/yangshixinwen_default.png'
      },
      topic_info: {
        tab: 'worldcup',
        boards: ['世界杯']
      },
      selector_info: {
        box: '.con > ul > li',
        title: '.text > a',
        content: '.text > a',
        pic: '.image img'
      }
    }

    let topics = await this.ctx.service.commonSvc.commonFetch(param);
    return topics;
  }


  async meituanjishu() {
    const param = {
      url: 'https://tech.meituan.com/',
      author_info: {
        author_id: '628ada0bb3ed682a5d9ab72e',
        author_name: '美团技术',
        author_avatar: 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/meituanjishu.jpeg'
      },
      topic_info: {
        tab: 'tech',
        boards: ['架构']
      },
      selector_info: {
        box: '.col-md-6',
        title: '.post-title > a',
        content: '.post-title > a',
        intro: '.post-content.post-expect'
      }
    }

    let topics = await this.ctx.service.commonSvc.commonLight(param);
    return topics;
  }


  async dbaplus() {
    const param = {
      url: 'https://dbaplus.cn/',
      author_info: {
        author_id: '62d01b57884ef71e1683183a',
        author_name: 'dbaplus社群',
        author_avatar: 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/dbaplus.jpg'
      },
      topic_info: {
        tab: 'tech',
        boards: ['架构']
      },
      selector_info: {
        box: 'ul#tabAll > li.media',
        title: '.media-heading > a',
        content: '.media-heading > a',
        pic: 'img.media-object',
        intro: '.mt10.geay'
      }
    }

    let topics = await this.ctx.service.commonSvc.commonLight(param);
    return topics;
  }


  async meirixiache() {
    const param = {
      url: 'https://daily.zhihu.com/',
      author_info: {
        author_id: '5f886a8a6a591b09964cd4c3',
        author_name: '每日瞎扯',
        author_avatar: '/public/images/xiache.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['搞笑']
      },
      selector_info: {
        box: '.wrap',
        title: 'a',
        content: 'a',
        pic: 'img'
      }
    }

    let topics = await this.ctx.service.commonSvc.commonLight(param);
    topics = topics.filter(v => v.title.includes('如何正确地吐槽'));
    topics.forEach(v => {
      let tmp = v.content.split('/');
      v.title += '-' + tmp[tmp.length - 1];
    })
    return topics;
  }

  async zhongguogequpaihangbang() {
    const {ctx} = this;

    let totalTopics = [];

    const url = 'https://mobile.tingtingfm.com/v3/program/q0boxOndrg';

    const tab = 'podcast';
    const author_id = '6304e647d5046112773ed8f7';
    const author_name = '中国歌曲排行榜';
    const author_avatar = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/zhongguogequpaihangbang.jpg';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.list-num > a.link');
      const eles = await page.$$('.list-num > a.link');

      for (const el of eles) {
        let title = await el.$eval('.name', v => v.innerText);
        let content = await el.evaluate(async v => {
          return v.outerHTML;
        });

        content = content.replace(/.+href="(.+?)".+/, 'https://mobile.tingtingfm.com$1');

        let topic = {
          title,
          content,
          original_url: content,
          pic: 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/zhongguogequpaihangbang.jpg',
          boards: ['音乐'],
          tags: [],
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error('马蜂窝游记爬取异常!!!');
      }
      return totalTopics;
    }

  }

  async jinpingmei() {
    const {ctx} = this;

    let param = {
      url: 'https://podcasts.apple.com/cn/podcast/%E9%87%91%E7%93%B6%E6%A2%85/id1551581507',
      author_info: {
        author_id: '6304df4e2e655d10d0a90184',
        author_name: '金瓶梅-船多不碍港',
        author_avatar: 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/jinpingmei.jpg'
      },
      topic_info: {
        tab: 'podcast',
        boards: ['播客']
      },
      selector_info: {
        box: 'li.tracks__track.tracks__track--podcast',
        title: 'h2',
        content: 'a',
      }
    };

    let result = await ctx.service.commonSvc.commonFetch(param);
    return result;

  }

  getTagBoardMapping(tag) {
    const tagBoardMapping = {
      '-': ['社会文化'],
      '科技': ['科技'],
      '爱好': ['趣味闲谈'],
      '表演': ['影视娱乐'],
      '创业': ['商业'],
      '商务': ['商业'],
      '地球科学': ['科技'],
      '电视与电影': ['影视娱乐'],
      '个人日记': ['社会文化'],
      '基督教': ['社会文化'],
      '社会与文化': ['社会文化'],
      '非营利': ['商业'],
      '管理': ['职场'],
      '休闲': ['趣味闲谈'],
      '职业': ['职场'],
      '即兴表演': ['趣味闲谈'],
      '家居与园艺': ['趣味闲谈'],
      '电子游戏': ['游戏'],
      '食物': ['美食'],
      '独角喜剧': ['艺术'],
      '喜剧专访': ['艺术'],
      '剧情': ['艺术'],
      '科幻小说': ['艺术'],
      '喜剧小说': ['艺术'],
      '小说': ['艺术'],
      '营养': ['医疗健康'],
      '跑步': ['体育运动'],
      '另类保健': ['医疗健康'],
      '健美': ['医疗健康'],
      '两性': ['医疗健康'],
      '医学': ['医疗健康'],
      '足球': ['体育运动'],
      '篮球': ['体育运动'],
      '野外': ['体育运动'],
      '游泳': ['体育运动'],
      '语言学习': ['自我成长'],
      '指南': ['自我成长'],
      '科技新闻': ['科技'],
      '情感关系': ['社会文化'],
      '新闻解说': ['社会文化'],
      '投资': ['商业'],
      '影评': ['影视娱乐'],
      '电影专访': ['影视娱乐'],
      '社会科学': ['社会文化'],
      '自然科学': ['科技'],
      '商业新闻': ['商业'],
      '营销': ['商业'],
      '音乐赏析': ['音乐'],
      '音乐史': ['音乐'],
      '音乐专访': ['音乐'],
      '心灵修炼': ['治愈陪伴'],
      '灵修': ['自我成长'],
      '时尚美容': ['治愈陪伴'],
      '视觉艺术': ['艺术'],
      '电视评论': ['影视娱乐'],
      '节目回顾': ['影视娱乐'],
      '儿童教育': ['社会文化'],
      '宠物与动物': ['儿童与家庭'],
      '动画与漫画': ['动漫游戏'],
      '历史': ['自我成长'],
      '哲学': ['自我成长'],
      '宗教与心灵': ['自我成长']
    };

    return tagBoardMapping[tag] || [tag];
  }


  async createPodcastUser(loginname, tag, url, rss) {
    const {ctx} = this;

    const is_exist_user = await ctx.model.User.findOne({'froms.url': url});
    if (is_exist_user) {
      console.error('不正常的用户:', loginname, url);
      is_exist_user.loginname = is_exist_user.nickname = loginname;
      await is_exist_user.save();
      return;
    }

    let default_boards = this.getTagBoardMapping(tag);
    let froms = [{name: '中文播客榜', url: url}];
    if (rss) {
      froms.push({name: '播客RSS', url: rss});
    }

    const browser = await chromium.launch({headless: true});
    try {
      let page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});

      await page.goto(url, {waitUntil: 'domcontentloaded', timeout: 5000});
      await page.waitForSelector('ul.tab > li');

      let avatarUrl = await page.$eval('main > div.wrap > div.avatar-wrapper > img.avatar', el => el.src);
      await page.click('main > main > div.labels > label:last-of-type');
      let signature = await page.$eval('main > main > div.tabs-container > div > div', el => el.innerText);


      let email = `${loginname}@taosha.club`;
      let pass = 'zzz';
      let passhash = ctx.helper.bhash(pass);
      let result = await ctx.service.userSvc.newAndSave(loginname, loginname, passhash, email, avatarUrl, true, url, signature, default_boards, froms);

      console.log(result['_doc']['_id'], result['_doc']['loginname']);
    } catch (e) {
      console.error(e.message);
      await ctx.model.User.findOneAndUpdate({loginname}, {$set: {froms}});
    } finally {
      await browser.close();
    }
  }

  async juejinAll() {
    this.logger.info('juejinAll 开始');
    const {ctx} = this;

    let r1 = await ctx.service.crawlerSvc.juejin('https://juejin.cn/backend?sort=three_days_hottest', ['后端']);
    let r2 = await ctx.service.crawlerSvc.juejin('https://juejin.cn/frontend?sort=three_days_hottest', ['前端']);
    let r4 = await ctx.service.crawlerSvc.juejin('https://juejin.cn/ai?sort=three_days_hottest', ['AI·大数据']);
    let r3 = await ctx.service.crawlerSvc.juejin('https://juejin.cn/freebie?sort=three_days_hottest', ['其他']);
    let r5 = await ctx.service.crawlerSvc.juejin('https://juejin.cn/career?sort=three_days_hottest', ['其他']);
    let r6 = await ctx.service.crawlerSvc.juejin('https://juejin.cn/article?sort=three_days_hottest', ['其他']);

    let topics = r1
      .concat(r2)
      .concat(r3)
      .concat(r4)
      .concat(r5)
      .concat(r6)
    ;

    topics = ctx.helper.betterUniqueTopics(topics);

    ctx.logger.info('juejinAll', topics.length);
    return topics;
  }


  async oschinaAll() {
    this.logger.info('oschinaAll 开始');
    const {ctx} = this;

    let topics1 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=8&p=1&type=ajax', ['AI·大数据']);
    let topics2 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=9&p=1&type=ajax', ['其他']);
    let topics3 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=10&p=1&type=ajax', ['数据库']);
    let topics4 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=13&p=1&type=ajax', ['其他']);
    let topics5 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=14&p=1&type=ajax', ['其他']);
    let topics6 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=19&p=1&type=ajax', ['其他']);
    let topics7 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=22&p=1&type=ajax', ['其他']);
    let topics8 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=23&p=1&type=ajax', ['测试']);
    let topics9 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=24&p=1&type=ajax', ['其他']);
    let topics10 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=26&p=1&type=ajax', ['架构']);
    let topics11 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=26&p=1&type=ajax', ['后端']);
    let topics12 = await ctx.service.crawlerSvc.oschina('https://www.oschina.net/blog/widgets/_blog_index_recommend_list?classification=28&p=1&type=ajax', ['前端']);

    let topics = topics1
      .concat(topics2)
      .concat(topics3)
      .concat(topics4)
      .concat(topics5)
      .concat(topics6)
      .concat(topics7)
      .concat(topics8)
      .concat(topics9)
      .concat(topics10)
      .concat(topics11)
      .concat(topics12)
    ;

    this.logger.info('oschinaAll ready', topics.length);
    return topics;
  }


  async infoqAll() {
    const {ctx} = this;

    let topics1 = await ctx.service.crawlerSvc.infoq('https://www.infoq.cn/topic/architecture', ['架构']);
    let topics2 = await ctx.service.crawlerSvc.infoq('https://www.infoq.cn/topic/1174', ['后端']);
    let topics3 = await ctx.service.crawlerSvc.infoq('https://www.infoq.cn/topic/1208', ['前端']);
    let topics4 = await ctx.service.crawlerSvc.infoq('https://www.infoq.cn/topic/1168', ['其他']);
    let topics5 = await ctx.service.crawlerSvc.infoq('https://www.infoq.cn/topic/1204', ['其他']);
    let topics6 = await ctx.service.crawlerSvc.infoq('https://www.infoq.cn/topic/1195', ['其他']);
    let topics7 = await ctx.service.crawlerSvc.infoq('https://www.infoq.cn/topic/AI', ['AI·大数据']);
    let topics8 = await ctx.service.crawlerSvc.infoq('https://www.infoq.cn/topic/bigdata', ['AI·大数据']);
    let topics9 = await ctx.service.crawlerSvc.infoq('https://www.infoq.cn/topic/cloud-computing', ['其他']);
    let topics10 = await ctx.service.crawlerSvc.infoq('https://www.infoq.cn/topic/Database', ['数据库']);


    let totalTopics = topics1
      .concat(topics2)
      .concat(topics3)
      .concat(topics4)
      .concat(topics5)
      .concat(topics6)
      .concat(topics7)
      .concat(topics8)
      .concat(topics9)
      .concat(topics10)
    ;

    totalTopics = ctx.helper.betterUniqueTopics(totalTopics);

    return totalTopics;
  }

  async wuyaoctoAll() {
    const {ctx} = this;

    let topics1 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/developer', ['后端']);
    let topics3 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/test', ['测试']);
    let topics4 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/architecture', ['架构']);
    let topics5 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/tool', ['其他']);
    let topics6 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/ai', ['AI·大数据']);
    let topics7 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/cloud', ['其他']);
    let topics8 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/opensource', ['其他']);
    let topics9 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/translation', ['其他']);
    let topics10 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/bigdata', ['AI·大数据']);
    let topics11 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/network', ['其他']);
    let topics12 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/database', ['数据库']);
    let topics13 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/iot', ['其他']);
    let topics14 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/operation', ['其他']);
    let topics15 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/original', ['其他']);
    let topics16 = await ctx.service.crawlerSvc.wuyaocto('https://www.51cto.com/digital', ['其他']);


    let totalTopics = topics1
      .concat(topics3)
      .concat(topics4)
      .concat(topics5)
      .concat(topics6)
      .concat(topics7)
      .concat(topics8)
      .concat(topics9)
      .concat(topics10)
      .concat(topics11)
      .concat(topics12)
      .concat(topics13)
      .concat(topics14)
      .concat(topics15)
      .concat(topics16)
    ;

    totalTopics = ctx.helper.betterUniqueTopics(totalTopics);
    for (let topic of totalTopics) {
      if (!topic.pic) {
        continue;
      }
      topic.pic = topic.pic.replace(/\?x-oss-process.+/, '');
    }

    return totalTopics;
  }

  async jiemianAll() {
    const {ctx} = this;

    let param = {
      author_info: {
        author_id: '6219cae43a07421946682580',
        author_name: '界面新闻',
        author_avatar: '/public/images/jiemian.png'
      },
      topic_info: {
        tab: 'yutang',
      },
      selector_info: {
        box: 'li.card-list',
        title: '.card-list__title',
        content: '.card-list__content a',
        pic: '.news-img img',
        intro: '.card-list__summary'
      }
    };


    param.url = 'https://www.jiemian.com/lists/65.html';
    param.topic_info.boards = ['科技'];
    let r1 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/62.html';
    param.topic_info.boards = ['房产'];
    let r2 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/51.html';
    param.topic_info.boards = ['汽车'];
    let r3 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/31.html';
    param.topic_info.boards = ['消费'];
    let r4 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/28.html';
    param.topic_info.boards = ['工业'];
    let r5 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/30.html';
    param.topic_info.boards = ['交通'];
    let r6 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/472.html';
    param.topic_info.boards = ['医药'];
    let r7 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/851.html';
    param.topic_info.boards = ['互联网'];
    let r8 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/858.html';
    param.topic_info.boards = ['创投'];
    let r9 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/856.html';
    param.topic_info.boards = ['能源'];
    let r10 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/853.html';
    param.topic_info.boards = ['数码'];
    let r11 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/256.html';
    param.topic_info.boards = ['教育'];
    let r12 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/845.html';
    param.topic_info.boards = ['食品'];
    let r13 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/854.html';
    param.topic_info.boards = ['健康'];
    let r14 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/841.html';
    param.topic_info.boards = ['物流'];
    let r15 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/847.html';
    param.topic_info.boards = ['零售'];
    let r16 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/city/main/181.html';
    param.topic_info.boards = ['房产'];
    let r17 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/848.html';
    param.topic_info.boards = ['餐饮'];
    let r18 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/852.html';
    param.topic_info.boards = ['企业'];
    let r19 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/605.html';
    param.topic_info.boards = ['商业'];
    let r20 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/800.html';
    param.topic_info.boards = ['财经'];
    let r21 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/32.html';
    param.topic_info.boards = ['国际'];
    let r22 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/8.html';
    param.topic_info.boards = ['评论'];
    let r23 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/154.html';
    param.topic_info.boards = ['数据'];
    let r24 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://www.jiemian.com/lists/50.html';
    param.topic_info.boards = ['职场'];
    let r25 = await ctx.service.commonSvc.commonLight(param);


    param = {
      url: 'https://www.jiemian.com/lists/130.html',
      author_info: {
        author_id: '6219cae43a07421946682580',
        author_name: '界面新闻',
        author_avatar: '/public/images/jiemian.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['文化']
      },
      selector_info: {
        box: '.card-list',
        title: 'a > h3',
        content: 'a',
        pic: 'img',
        intro: '.card-list__summary'
      },
      top_selector_info: {
        box: '.full-big-img__item',
        title: 'h3 > a',
        content: 'h3 > a',
        pic: 'img',
      }
    };

    let r26 = await ctx.service.commonSvc.commonLight(param);


    param = {
      url: 'https://www.jiemian.com/account/main/1.html',
      author_info: {
        author_id: '6219cae43a07421946682580',
        author_name: '界面新闻',
        author_avatar: '/public/images/jiemian.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['界面号', '财经']
      },
      top_selector_info: {
        box: '.swiper-slide',
        title: 'a',
        content: 'a',
        pic: 'img',
      },
      selector_info: {
        box: 'li.item',
        title: '.title a',
        content: '.title a',
        pic: 'img',
      }
    };

    let r27 = await ctx.service.commonSvc.commonLight(param);


    param.url = 'https://www.jiemian.com/account/main/2.html';
    param.topic_info.boards = ['界面号'];
    let r28 = await ctx.service.commonSvc.commonLight(param);


    param.url = 'https://www.jiemian.com/account/main/3.html';
    param.topic_info.boards = ['界面号'];
    let r29 = await ctx.service.commonSvc.commonLight(param);


    param = {
      url: 'https://www.jiemian.com/video/lists/index_1.html',
      author_info: {
        author_id: '6219cae43a07421946682580',
        author_name: '界面新闻',
        author_avatar: '/public/images/jiemian.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['视频']
      },
      top_selector_info: {
        box: '.swiper-slide',
        title: 'h3 > a',
        content: 'h3 > a',
        pic: 'img',
      },
      selector_info: {
        box: 'li.card-list',
        title: '.card-list__content a.logStore',
        content: '.card-list__content a.logStore',
        pic: 'img',
        intro: '.card-list__summary'
      }
    };

    let r30 = await ctx.service.commonSvc.commonLight(param);


    let topics = r1
      .concat(r2)
      .concat(r3)
      .concat(r4)
      .concat(r5)
      .concat(r6)
      .concat(r7)
      .concat(r8)
      .concat(r9)
      .concat(r10)
      .concat(r11)
      .concat(r12)
      .concat(r13)
      .concat(r14)
      .concat(r15)
      .concat(r16)
      .concat(r17)
      .concat(r18)
      .concat(r19)
      .concat(r20)
      .concat(r21)
      .concat(r22)
      .concat(r23)
      .concat(r24)
      .concat(r25)
      .concat(r26)
      .concat(r27)
      .concat(r28)
      .concat(r29)
      .concat(r30)
    ;

    topics = ctx.helper.betterUniqueTopics(topics);

    // 经过评估, 发现界面的内容更适合全部放在鱼塘版块
    topics.forEach(v => v.tab = 'yutang');

    return topics;
  }

  async huxiuAll() {
    const {ctx} = this;

    let param = {
      author_info: {
        author_id: '6219ca975933df18fafaf73e',
        author_name: '虎嗅新闻',
        author_avatar: '/public/images/huxiu.png'
      },
      topic_info: {
        tab: 'yutang',
      },
      selector_info: {
        box: '.channel > ul > li',
        title: 'h2.title-wrap.more-line-ellipsis',
        content: 'a',
        pic: 'img.img',
      },
      scroll: 2000
    };


    param.url = 'https://m.huxiu.com/channel/10.html';
    param.topic_info.boards = ['视频'];
    let r1 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/115.html';
    param.topic_info.boards = ['财经'];
    let r2 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/22.html';
    param.topic_info.boards = ['娱乐淘金', '传媒'];
    let r3 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/21.html';
    param.topic_info.boards = ['汽车'];
    let r4 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/106.html';
    param.topic_info.boards = ['年轻一代', '传媒'];
    let r5 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/103.html';
    param.topic_info.boards = ['消费者', '生活'];
    let r6 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/105.html';
    param.topic_info.boards = ['科技'];
    let r7 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/118.html';
    param.topic_info.boards = ['健康'];
    let r8 = await ctx.service.commonSvc.commonFetch(param);

    // param.url = 'https://m.huxiu.com/channel/113.html';
    // param.topic_info.boards = ['传媒'];
    // let r9 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/114.html';
    param.topic_info.boards = ['出海', '传媒'];
    let r10 = await ctx.service.commonSvc.commonFetch(param);

    // param.url = 'https://m.huxiu.com/channel/102.html';
    // param.topic_info.boards = ['金融地产', '房产'];
    // let r11 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/119.html';
    param.topic_info.boards = ['人文', '书影音'];
    let r12 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/120.html';
    param.topic_info.boards = ['健康', '医疗'];
    let r13 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/121.html';
    param.topic_info.boards = ['数码'];
    let r14 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/107.html';
    param.topic_info.boards = ['全球热点', '传媒'];
    let r15 = await ctx.service.commonSvc.commonFetch(param);

    // param.url = 'https://m.huxiu.com/channel/4.html';
    // param.topic_info.boards = ['生活'];
    // let r16 = await ctx.service.commonSvc.commonFetch(param);

    // param.url = 'https://m.huxiu.com/channel/122.html';
    // param.topic_info.boards = ['传媒', '观点'];
    // let r17 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.huxiu.com/channel/123.html';
    param.topic_info.boards = ['其他'];
    let r18 = await ctx.service.commonSvc.commonFetch(param);

    let r19 = await this.huxiuzhuanlan();


    let topics = r1
      .concat(r2)
      .concat(r3)
      .concat(r4)
      .concat(r5)
      .concat(r6)
      .concat(r7)
      .concat(r8)
      // .concat(r9)
      .concat(r10)
      // .concat(r11)
      .concat(r12)
      .concat(r13)
      .concat(r14)
      .concat(r15)
      // .concat(r16)
      // .concat(r17)
      .concat(r18)
      .concat(r19)
    ;

    topics = ctx.helper.betterUniqueTopics(topics);

    topics.forEach(v => {
      if (v.title.startsWith('原创')) {
        v.title = v.title.replace('原创', '');
      }
      if (v.pic) {
        v.pic = v.pic.replace('/w/180/h/144/|imageMogr2/strip/interlace/1/quality/85', '/w/500/h/400');
      }
      v.content = v.original_url = v.content.replace('https://m.huxiu.com/', 'https://www.huxiu.com/');
    })

    return topics;
  }


  async huxiuzhuanlan() {
    let totalTopics = [];

    const url = 'https://pro.huxiu.com/vipColumn/new';

    const tab = 'yutang';
    const author_id = '6219ca975933df18fafaf73e';
    const author_name = '虎嗅新闻';
    const author_avatar = '/public/images/huxiu.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.column__list > div');
      const eles = await page.$$('.column__list > div');
      await this.ctx.helper.autoScrollByLimit(page, 1000);

      for (const el of eles) {
        let title = await el.$eval('.title', v => v.innerText);
        let content = await el.$eval('div', v => v.parentNode.outerHTML);
        content = content.match(/&quot;(\d{6,})&quot;/);
        content = 'https://pro.huxiu.com/article/' + content[1];

        let pic = await el.$eval('.article__img > img', v => v.src);

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          boards: ['专栏', '传媒'],
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error('虎嗅专栏爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async qqAll() {
    const {ctx} = this;

    const url = 'https://r.inews.qq.com/web_feed/getWebList';

    const data = {
      'device_id': '1_276318296',
      'channel_id': 'news_news_football',
      'base_req': {
        'from': 'qq_web'
      },
    };

    let resp = await got.post(url, {json: data});
    const r1 = this._getQQTopics(resp.data.data, '体育');

    data.channel_id = 'news_news_mil';
    resp = await got.post(url, {json: data});
    const r2 = this._getQQTopics(resp.data.data, '军事');

    data.channel_id = 'news_news_world';
    resp = await got.post(url, {json: data});
    const r3 = this._getQQTopics(resp.data.data, '新闻');

    data.channel_id = 'news_news_tech';
    resp = await got.post(url, {json: data});
    const r4 = this._getQQTopics(resp.data.data, '科技');

    data.channel_id = 'news_news_finance';
    resp = await got.post(url, {json: data});
    const r5 = this._getQQTopics(resp.data.data, '财经');

    data.channel_id = 'news_news_auto';
    resp = await got.post(url, {json: data});
    const r6 = this._getQQTopics(resp.data.data, '汽车');

    data.channel_id = 'news_news_digi';
    resp = await got.post(url, {json: data});
    const r7 = this._getQQTopics(resp.data.data, '数码');

    data.channel_id = 'news_news_antip';
    resp = await got.post(url, {json: data});
    const r8 = this._getQQTopics(resp.data.data, '健康');

    data.channel_id = 'news_news_edu';
    resp = await got.post(url, {json: data});
    const r9 = this._getQQTopics(resp.data.data, '教育');

    data.channel_id = 'news_news_history';
    resp = await got.post(url, {json: data});
    const r10 = this._getQQTopics(resp.data.data, '历史');

    data.channel_id = 'news_news_kepu';
    resp = await got.post(url, {json: data});
    const r11 = this._getQQTopics(resp.data.data, '百科');

    data.channel_id = 'news_news_baby';
    resp = await got.post(url, {json: data});
    const r12 = this._getQQTopics(resp.data.data, '亲子');

    data.channel_id = 'news_news_fx';
    resp = await got.post(url, {json: data});
    const r13 = this._getQQTopics(resp.data.data, '人文');

    data.channel_id = 'news_news_house';
    resp = await got.post(url, {json: data});
    const r14 = this._getQQTopics(resp.data.data, '房产');

    const hot_url = 'https://r.inews.qq.com/news_feed/hot_module_list';
    const hot_data = {
      'devid': '1_276318296',
      'chlid': 'news_news_football'
    };
    let hot_response = await got(hot_url, {searchParams: hot_data});
    const h1 = this._getHotQQTopics(hot_response.data.newslist, '体育');

    hot_data.chlid = 'news_news_mil';
    hot_response = await got(hot_url, {searchParams: hot_data});
    const h2 = this._getHotQQTopics(hot_response.data.newslist, '军事');

    hot_data.chlid = 'news_news_world';
    hot_response = await got(hot_url, {searchParams: hot_data});
    const h3 = this._getHotQQTopics(hot_response.data.newslist, '新闻');

    hot_data.chlid = 'news_news_finance';
    hot_response = await got(hot_url, {searchParams: hot_data});
    const h4 = this._getHotQQTopics(hot_response.data.newslist, '财经');

    hot_data.chlid = 'news_news_auto';
    hot_response = await got(hot_url, {searchParams: hot_data});
    const h5 = this._getHotQQTopics(hot_response.data.newslist, '汽车');

    hot_data.chlid = 'news_news_health';
    hot_response = await got(hot_url, {searchParams: hot_data});
    const h6 = this._getHotQQTopics(hot_response.data.newslist, '健康');

    hot_data.chlid = 'news_news_edu';
    hot_response = await got(hot_url, {searchParams: hot_data});
    const h7 = this._getHotQQTopics(hot_response.data.newslist, '教育');

    hot_data.chlid = 'news_news_house';
    hot_response = await got(hot_url, {searchParams: hot_data});
    const h8 = this._getHotQQTopics(hot_response.data.newslist, '房产');

    hot_data.chlid = 'news_news_history';
    hot_response = await got(hot_url, {searchParams: hot_data});
    const h9 = this._getHotQQTopics(hot_response.data.newslist, '历史');

    hot_data.chlid = 'news_news_baby';
    hot_response = await got(hot_url, {searchParams: hot_data});
    const h10 = this._getHotQQTopics(hot_response.data.newslist, '亲子');

    hot_data.chlid = 'news_news_fx';
    hot_response = await got(hot_url, {searchParams: hot_data});
    const h11 = this._getHotQQTopics(hot_response.data.newslist, '人文');


    let topics = r1
      .concat(r2)
      .concat(r3)
      .concat(r4)
      .concat(r5)
      .concat(r6)
      .concat(r7)
      .concat(r8)
      .concat(r9)
      .concat(r10)
      .concat(r11)
      .concat(r12)
      .concat(r13)
      .concat(r14)
      .concat(h1)
      .concat(h2)
      .concat(h3)
      .concat(h4)
      .concat(h5)
      .concat(h6)
      .concat(h7)
      .concat(h8)
      .concat(h9)
      .concat(h10)
      .concat(h11)
    ;

    topics = ctx.helper.betterUniqueTopics(topics);

    this.ctx.helper.setDefaultPic(topics);

    topics.forEach(v => {
      if (v.pic) {
        v.pic = v.pic.replace('thumbnail=230', 'thumbnail=500')
          .replace('quality=75', 'quality=100');
      }
    })

    return topics;
  }

  _getQQTopics(list, board) {
    const source = '腾讯新闻' + dayjs().format('YYYY-MM-DD');
    const total_topics = [];
    try {
      for (let i = 0; i < list.length; i++) {
        let item = list[i];
        const topic = {
          title: item.title,
          content: item.link_info.url,
          original_url: item.link_info.url,
          pic: item.pic_info.big_img[0],
          author_id: '62ece0288e1f347a5c98bf7b',
          author_name: '腾讯新闻',
          author_avatar: 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/tengxunxinwen.jpg',
          source,
          tab: 'yutang',
          boards: [board],
          is_video: !!item.video_info
        }
        total_topics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
      this.ctx.helper.dd_warn(e);
    } finally {
      return total_topics;
    }
  }


  _getHotQQTopics(list, board) {
    if (!list) {
      return [];
    }
    const source = '腾讯新闻' + dayjs().format('YYYY-MM-DD');
    const total_topics = [];
    for (let i = 0; i < list.length; i++) {
      let item = list[i];
      const topic = {
        title: item.title,
        content: item.url,
        original_url: item.url,
        pic: item.miniProShareImage,
        author_id: '62ece0288e1f347a5c98bf7b',
        author_name: '腾讯新闻',
        author_avatar: 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/tengxunxinwen.jpg',
        source,
        tab: 'yutang',
        boards: [board],
        is_video: !!item.video_channel
      }
      total_topics.push(topic);
    }
    return total_topics;
  }


  async wangyiAll() {
    const {ctx} = this;

    let r4 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/news/sub/history', ['历史']);
    let r5 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/sports', ['体育']);
    let r6 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/money', ['财经']);
    let r7 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/auto', ['汽车']);
    let r8 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/war', ['军事']);
    let r9 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/tech', ['科技']);
    let r10 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/tech/sub/techscience', ['科技']);
    let r11 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/tech/sub/smart', ['科技']);
    let r12 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/mobile', ['数码']);
    let r13 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/digi', ['数码']);
    let r14 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/dy', ['网易号', '传媒']);
    let r15 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/edu', ['教育']);
    let r16 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/jiankang', ['健康']);
    let r17 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/travel', ['旅游']);
    let r19 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/art', ['人文', '艺术']);
    let r20 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/open/', ['公开课', '传媒']);
    let r21 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/open/sub/ted', ['公开课', '传媒']);
    let r22 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/open/sub/jpkc', ['公开课', '传媒']);
    let r23 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/shuangchuang', ['双创']);
    let r24 = await ctx.service.commonSvc.wangyi('https://m.163.com/touch/video', ['视频']);


    let topics = []
      .concat(r4)
      .concat(r5)
      .concat(r6)
      .concat(r7)
      .concat(r8)
      .concat(r9)
      .concat(r10)
      .concat(r11)
      .concat(r12)
      .concat(r13)
      .concat(r14)
      .concat(r15)
      .concat(r16)
      .concat(r17)
      .concat(r19)
      .concat(r20)
      .concat(r21)
      .concat(r22)
      .concat(r23)
      .concat(r24)
    ;

    topics = ctx.helper.betterUniqueTopics(topics);

    return topics;
  }


  async sohuAll() {
    const {ctx} = this;
    let param;

    param = {
      url: 'https://m.sohu.com/xchannel/tag?key=新闻-时政',
      is_mobile: true,
      author_info: {
        author_id: '62317f42b479c2424a0a0a19',
        author_name: '搜狐新闻',
        author_avatar: 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/sohu.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['新闻']
      },
      top_selector_info: {
        box: '.swiper-container > div > div',
        title: '.pics-title-cup-title,.title',
        content: 'a',
        pic: 'img'
      },
      selector_info: {
        box: '.FeedList > div',
        title: '.item-text-content-title,.title',
        content: 'a',
        pic: 'img',
        is_video: '.video'
      },
      scroll: 3000
    };
    let r1 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/xchannel/tag?key=新闻-国际';
    param.topic_info.boards = ['新闻'];
    let r2 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/xchannel/tag?key=新闻-财经';
    param.topic_info.boards = ['财经'];
    let r3 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/xchannel/tag?key=体育';
    param.topic_info.boards = ['体育'];
    let r4 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/mil';
    param.topic_info.boards = ['军事'];
    let r5 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://www.sohu.com/xchannel/TURBd01EQXdOekUx';
    param.topic_info.boards = ['健康'];
    let r6 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/history';
    param.topic_info.boards = ['历史'];
    let r7 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/xchannel/tag?key=科技-互联网';
    param.topic_info.boards = ['科技'];
    let r8 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/media';
    param.topic_info.boards = ['传媒'];
    let r9 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/chihe';
    param.topic_info.boards = ['美食'];
    let r10 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/travel';
    param.topic_info.boards = ['旅游'];
    let r11 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://www.sohu.com/xchannel/TURBd01EQXdOekUx';
    param.topic_info.boards = ['健康'];
    let r12 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/baobao';
    param.topic_info.boards = ['亲子'];
    let r13 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/xchannel/tag?key=科技-数码';
    param.topic_info.boards = ['数码'];
    let r14 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/xchannel/TURBd01EQXdOVEEw';
    param.topic_info.boards = ['数码'];
    let r15 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.sohu.com/learning';
    param.topic_info.boards = ['教育'];
    let r17 = await ctx.service.commonSvc.commonFetch(param);


    let topics = r1
      .concat(r2)
      .concat(r3)
      .concat(r4)
      .concat(r5)
      .concat(r6)
      .concat(r7)
      .concat(r8)
      .concat(r9)
      .concat(r10)
      .concat(r11)
      .concat(r12)
      .concat(r13)
      .concat(r14)
      .concat(r15)
      .concat(r17)
    ;

    topics = ctx.helper.betterUniqueTopics(topics);
    topics.forEach(v => {
      v.content = v.original_url = v.content.replace('https://m.sohu.com/', 'https://www.sohu.com/');
    })

    ctx.helper.setDefaultPic(topics);

    return topics;
  }


  async guanchazheAll() {
    const {ctx} = this;

    let param = {
      url: 'https://www.guancha.cn/GuanChaZheTouTiao/list_1.shtml',
      author_info: {
        author_name: '观察者网',
        author_id: '6215027814ead414b2000280',
        author_avatar: '/public/images/guancha.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['热点']
      },
      selector_info: {
        box: '.headline-list > li',
        title: '.content-headline h3 a',
        content: '.content-headline h3 a',
        pic: 'a img',
        intro: '.resemble-art,.module-artile'
      }
    };

    let r1 = await ctx.service.commonSvc.commonLight(param);


    param = {
      url: 'https://www.guancha.cn/',
      author_info: {
        author_name: '观察者网',
        author_id: '6215027814ead414b2000280',
        author_avatar: '/public/images/guancha.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['热点']
      },
      selector_info: {
        box: 'ul > li',
        title: 'h4 > a',
        content: 'h4 > a',
        pic: '.fastRead-img img,.fengwen-list-img img,.module-img > img,.static-img img',
        intro: '.resemble-art,.module-artile',
      },
      scroll: 10000
    };

    let r2 = await ctx.service.commonSvc.commonFetch(param);


    param = {
      url: 'https://www.guancha.cn/economy?s=dhcaijing',
      author_info: {
        author_name: '观察者网',
        author_id: '6215027814ead414b2000280',
        author_avatar: '/public/images/guancha.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['财经'],
      },
      selector_info: {
        box: 'ul > li',
        title: 'h4 > a',
        content: 'h4 > a',
        pic: 'a img',
        intro: '.resemble-art,.module-artile',
      },
    };

    let r3 = await ctx.service.commonSvc.commonLight(param);


    param = {
      url: 'https://www.guancha.cn/internation?s=dhguoji',
      author_info: {
        author_name: '观察者网',
        author_id: '6215027814ead414b2000280',
        author_avatar: '/public/images/guancha.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['国际'],
      },
      selector_info: {
        box: 'ul > li',
        title: 'h4 > a',
        content: 'h4 > a',
        pic: '.fastRead-img img,.fengwen-list-img img,.module-img > img,.static-img img',
        intro: '.resemble-art,.module-artile',
      },
    };

    let r4 = await ctx.service.commonSvc.commonLight(param);

    param = {
      url: 'https://www.guancha.cn/military-affairs?s=dhjunshi',
      author_info: {
        author_name: '观察者网',
        author_id: '6215027814ead414b2000280',
        author_avatar: '/public/images/guancha.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['军事'],
      },
      selector_info: {
        box: 'ul > li',
        title: 'h4 > a',
        content: 'h4 > a',
        pic: '.fastRead-img img,.fengwen-list-img img,.module-img > img,.static-img img',
        intro: '.resemble-art,.module-artile',
      },
    };

    let r5 = await ctx.service.commonSvc.commonLight(param);


    let r9 = await this.ctx.service.crawlerSvc.fengwen();
    for (let i = 0; i < r9.length; i++) {
      let topic = r9[i];
      if (!topic.pic) {
        continue;
      }
      topic.pic = topic.pic.replace(/(.+)\?image.+/, '$1');
      topic.pic += '?imageView2/2/w/500/interlace/1';
    }


    let topics = r3
      .concat(r4)
      .concat(r5)
      .concat(r9)
      .concat(r1)
      .concat(r2)
    ;

    topics = this.ctx.helper.betterUniqueTopics(topics);

    topics.forEach(t => {
      if (!t.pic && !t.tags.includes('风闻')) {
        t.tags.push('评论');
      }
    })


    for (const topic of topics) {
      if (topic.content.includes('user.guancha.cn')) {
        topic.tab = 'yutang';
        topic.boards.unshift('风闻');
        topic.tags.unshift('风闻');
      }
      if (topic.intro) {
        topic.intro = topic.intro.replace('[全文]', '');
      }
      if (!topic.intro) {
        topic.intro = undefined;
      }

      if (!topic.pic) {
        continue;
      }
      if (topic.pic.includes('/authors/')) {
        topic.pic = undefined;
      } else if (topic.pic.includes('homepage-special-arrow')) {
        topic.pic = undefined;
      } else if (topic.pic.includes('https://i.guancha.cn/hot.png')) {
        topic.pic = undefined;
      }
    }

    return topics;
  }

  async pengpaiAll() {
    const {ctx} = this;
    let param;

    param = {
      author_info: {
        author_id: '6219ca83af06e718ebbcbbd8',
        author_name: '澎湃新闻',
        author_avatar: '/public/images/pengpai.png'
      },
      topic_info: {
        tab: 'yutang',
      },
      top_selector_info: {
        box: '.adm-swiper-slide',
        title: '.index_headline_swpier_title__4Wb3U > span',
        content: 'a',
        pic: 'img',
      },
      selector_info: {
        box: '.index_wrapper__9rz3z',
        title: 'h3',
        content: 'a',
        pic: 'img',
        is_advert: '.index_item_label__RwoaG:nth-of-type(2)'
      },
    };

    param.url = 'https://m.thepaper.cn/channel_25950';
    param.topic_info.boards = ['新闻'];
    let r1 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/channel_25951';
    param.topic_info.boards = ['财经'];
    let r2 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/channel_122908';
    param.topic_info.boards = ['新闻'];
    let r3 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/channel_119908';
    param.topic_info.boards = ['科技'];
    let r4 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/channel_25952';
    param.topic_info.boards = ['思想', '传媒'];
    let r5 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/channel_119489';
    param.topic_info.boards = ['智库', '传媒'];
    let r6 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/list_25448';
    param.topic_info.boards = ['电影'];
    let r7 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/list_26609';
    param.topic_info.boards = ['生活'];
    let r8 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/list_25942';
    param.topic_info.boards = ['健康'];
    let r9 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/list_25599';
    param.topic_info.boards = ['体育'];
    let r10 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/list_26862';
    param.topic_info.boards = ['房产'];
    let r12 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/list_25769';
    param.topic_info.boards = ['生活'];
    let r13 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/list_25990';
    param.topic_info.boards = ['传媒'];
    let r14 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/list_26173';
    param.topic_info.boards = ['传媒'];
    let r15 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/list_26202';
    param.topic_info.boards = ['亲子'];
    let r16 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/list_26490';
    param.topic_info.boards = ['汽车'];
    let r17 = await ctx.service.commonSvc.commonLight(param);

    param.url = 'https://m.thepaper.cn/list_115327';
    param.topic_info.boards = ['人文'];
    let r18 = await ctx.service.commonSvc.commonLight(param);


    param = {
      url: 'https://m.thepaper.cn/channel_26916',
      author_info: {
        author_id: '6219ca83af06e718ebbcbbd8',
        author_name: '澎湃新闻',
        author_avatar: '/public/images/pengpai.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['视频']
      },
      selector_info: {
        box: '.index_wrapper__QYLPr',
        title: 'a.index_inherit__wl1Pg',
        content: 'a.index_inherit__wl1Pg',
        pic: 'img',
      },
    };
    let r19 = await ctx.service.commonSvc.commonLight(param);

    param = {
      url: 'https://m.thepaper.cn/channel_36079',
      author_info: {
        author_id: '6219ca83af06e718ebbcbbd8',
        author_name: '澎湃新闻',
        author_avatar: '/public/images/pengpai.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['湃客', '传媒']
      },
      selector_info: {
        box: '.FirstCard_wrap__oRseG',
        title: 'a.FirstCard_item_title__qhT1A',
        content: 'a.FirstCard_item_title__qhT1A',
        pic: '.FirstCard_item_thumb__L6Kv_ > img',
      },
    };
    let r20 = await ctx.service.commonSvc.commonFetch(param);

    param.url = 'https://m.thepaper.cn/channel_77286';
    param.topic_info.boards = ['传媒'];
    let r21 = await ctx.service.commonSvc.commonFetch(param);


    const url = 'https://www.thepaper.cn/user_3454103';
    const boards = ['百科'];
    const r22 = await ctx.service.crawlerSvc.pengpaihao(url, boards);


    let topics = r1
      .concat(r2)
      .concat(r3)
      .concat(r4)
      .concat(r5)
      .concat(r6)
      .concat(r7)
      .concat(r8)
      .concat(r9)
      .concat(r10)
      .concat(r12)
      .concat(r13)
      .concat(r14)
      .concat(r15)
      .concat(r16)
      .concat(r17)
      .concat(r18)
      .concat(r19)
      .concat(r20)
      .concat(r21)
      .concat(r22)
    ;

    topics = this.ctx.helper.betterUniqueTopics(topics);

    topics.forEach(t => {
      t.content = t.original_url = t.content.replace('https://m.thepaper.cn', 'https://www.thepaper.cn');
      if (t.title.includes('早餐湃')) {
        t.tags.push('早报');
      } else if (t.title.includes('晚安湃')) {
        t.tags.push('晚报');
      }
      if (!t.pic) {
        t.pic = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/pengpaixinwen-big.jpg';
      } else {
        if (t.pic.includes('?x-oss-process')) {
          t.pic = t.pic.replace('?x-oss-process=image/resize,w_332', '?x-oss-process=image/resize,w_500');
        } else {
          t.pic = t.pic + '?x-oss-process=image/resize,w_500';
        }
      }
    })

    for (let i = 0; i < topics.length; i++) {
      let topic = topics[i];
      if (topic.pic == 'https://file.thepaper.cn/www/v3/img/media_wwwbig_default_new.png?x-oss-process=image/resize,w_500') {
        topics.splice(i--, 1);
      }
    }

    return topics;
  }

  async pengpaihao(url, boards) {
    const {ctx} = this;

    const param = {
      url: url,
      author_info: {
        author_id: '6219ca83af06e718ebbcbbd8',
        author_name: '澎湃新闻',
        author_avatar: '/public/images/pengpai.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: [...boards],
      },
      selector_info: {
        box: '.ant-col.ant-col-8',
        title: 'h2',
        content: '.index_inherit__A1ImK',
        pic: 'img',
      },
    };
    let topics = await ctx.service.commonSvc.commonFetch(param);
    return topics;
  }


  async zakerAll() {
    const {ctx} = this;

    let r1 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/', ['新闻']);
    let r2 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/23317', ['独家', '传媒']);
    let r3 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/?pos=selected_article', ['精读', '传媒']);
    let r4 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/660', ['新闻']);
    let r5 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/7', ['汽车']);
    let r6 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/8', ['体育']);
    let r7 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/13', ['科技']);
    r7.forEach(v => {
      if (v.title.includes('机型')
        || v.title.includes('一加')
        || v.title.includes('荣耀')
        || v.title.includes('华为')
        || v.title.includes('小米')
        || v.title.includes('三星')
        || v.title.includes('苹果')
        || v.title.toLowerCase().includes('oppo')
        || v.title.toLowerCase().includes('vivo')
      ) {
        v.boards = ['数码'];
      }
    })
    let r8 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/2', ['新闻']);
    let r9 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/3', ['军事']);
    let r10 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/4', ['财经']);
    let r11 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/5', ['科技']);
    let r12 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/11542', ['数码']);
    // let r13 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/11', ['教育']);
    let r14 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/959', ['亲子']);
    let r15 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/1039', ['科技']);
    let r16 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/10386', ['美食']);
    let r17 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/10802', ['健康']);
    let r18 = await this.ctx.service.crawlerSvc.zaker('https://www.myzaker.com/channel/10530', ['电影']);
    let r19 = await this.ctx.service.crawlerSvc.zaker('http://www.myzaker.com/channel/14', ['新闻']);
    // let r20 = await this.ctx.service.crawlerSvc.zaker('http://www.myzaker.com/channel/6', ['房产']);


    let topics = r1
      .concat(r2)
      .concat(r3)
      .concat(r4)
      .concat(r5)
      .concat(r6)
      .concat(r7)
      .concat(r8)
      .concat(r9)
      .concat(r10)
      .concat(r11)
      .concat(r12)
      .concat(r14)
      .concat(r15)
      .concat(r16)
      .concat(r17)
      .concat(r18)
      .concat(r19)
    ;

    topics = this.ctx.helper.betterUniqueTopics(topics);

    for (let i = 0; i < topics.length; i++) {
      const topic = topics[i];
      if (topic.boards.includes('新闻') && !topic.pic) {
        topics.splice(i--, 1);
        continue;
      }
    }

    this.ctx.helper.setDefaultPic(topics);

    return topics;
  }

  async baijiahao() {
    const {ctx} = this;

    const tab = 'yutang';
    const users = await ctx.model.User.find({'froms.name': '百家号新闻'}).sort({_id: -1});
    let cnt = users.length;
    for (let i = 0; i < cnt; i++) {
      const user = users[i];
      let topics = await ctx.service.commonSvc.commonBaijiahao(user.loginname, tab);

      for (let j = 0; j < topics.length; j++) {
        const topic = topics[j];
        if (!topic.pic) {
          topics.splice(j--, 1);
          continue;
        }
      }

      let result = await ctx.service.topicSvc.importTopics(topics);
      ctx.logger.info(`${i}, ${cnt}, ${user.loginname} 百家号新闻 ready ${topics.length} success ${result}`);
    }
  }


  async idaily() {
    const {ctx} = this;

    let param = {
      url: 'http://idaily-cdn.idailycdn.com/api/list/v3/iphone/zh-hans?page=1&ver=iphone',
      author_info: {
        author_name: '每日环球视野',
        author_id: '62923d27ee26ba57e0b49fb7',
        author_avatar: '/public/images/idaily.png'
      },
      topic_info: {
        tab: 'yutang',
      },
      selector_info: {
        box: '',
        title: 'title_wechat_tml',
        content: 'link_share',
        pic: 'cover_landscape',
        intro: 'content',
        tags: 'tags',
      }
    };

    let topics = await ctx.service.commonSvc.commonJson(param);

    for (const topic of topics) {
      topic.title = topic.title.split(' - ')[0];
      if (topic.tags) {
        topic.tags = topic.tags[0].name.split(' · ');
        if (topic.tags.length == 2) {
          topic.tags = [topic.tags[1]];
        }
      }
    }

    return topics;
  }

  async digitaling() {
    let param = {
      url: 'https://www.digitaling.com/',
      author_info: {
        author_name: '数英每日',
        author_id: '621a29ad14299d2937298690',
        author_avatar: '/public/images/shuying.png',
      },
      topic_info: {
        tab: 'yutang',
      },
      selector_info: {
        box: '.clearfix.page_1',
        title: '.line_em > a',
        content: '.line_em > a',
        intro: 'p',
        pic: 'img',
        visit_count: 'a.inline_bk.mg_r_20.color_666:nth-child(3)'
      },
      scroll: 10000
    };

    let topics = await this.ctx.service.commonSvc.commonFetch(param);

    for (const topic of topics) {
      if (topic.pic && topic.pic.includes('default_img')) {
        topic.pic = undefined;
      }
    }

    return topics;
  }


  async longtengwang() {
    const {ctx} = this;

    let param = {
      url: 'http://www.ltaaa.cn/article',
      author_info: {
        author_id: '621acc609638e92d051b4bdb',
        author_name: '龙腾网',
        author_avatar: '/public/images/longtengwang.png',
      },
      topic_info: {
        tab: 'yutang',
      },
      selector_info: {
        box: '.wlist > li',
        title: '.li-title > a',
        content: '.li-title > a',
        pic: '.li-thumb > img',
        intro: '.dbody > p',
        visit_count: 'span.p2:nth-of-type(4)',
      }
    };
    let topics = await ctx.service.commonSvc.commonLight(param);

    // topics.forEach(v => {
    //   v.pic = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/longtengwang-big.png';
    // })

    return topics;
  }


  async bilibili() {
    const {ctx} = this;

    let param;

    param = {
      url: 'https://www.bilibili.com/v/popular/all',
      author_info: {
        author_id: '62178e8643ab5464a7b15312',
        author_name: 'Bilibili热榜',
        author_avatar: '/public/images/bilibili2.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['综合热门'],
        tags: ['综合热门'],
      },
      selector_info: {
        box: '.card-list > .video-card',
        title: '.video-card__info > p',
        content: '.video-card__content > a',
        pic: '.video-card__content img',
        tags: '.rcmd-tag.weak-tag',
        visit_count: '.like-text',
      },
      scroll: 4000
    };

    let r1 = await ctx.service.commonSvc.commonFetch(param);


    param = {
      url: 'https://www.bilibili.com/v/popular/weekly',
      author_info: {
        author_id: '62178e8643ab5464a7b15312',
        author_name: 'Bilibili热榜',
        author_avatar: '/public/images/bilibili2.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['每周必看'],
        tags: ['每周必看'],
      },
      selector_info: {
        box: '.video-list > .video-card',
        title: '.video-card__info > p',
        content: '.video-card__content > a',
        pic: '.video-card__content img',
        intro: '.weekly-hint',
        visit_count: '.like-text',
      },
      scroll: 4000
    };

    let r2 = await ctx.service.commonSvc.commonFetch(param);


    param = {
      url: 'https://www.bilibili.com/v/popular/rank/all',
      author_info: {
        author_id: '62178e8643ab5464a7b15312',
        author_name: 'Bilibili热榜',
        author_avatar: '/public/images/bilibili2.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['全站热榜'],
        tags: ['全站热榜'],
      },
      selector_info: {
        box: '.rank-list > .rank-item',
        title: '.info > a',
        content: '.info > a',
        pic: 'img',
        visit_count: '.detail-state > span:nth-of-type(2)',
      },
      scroll: 10000
    };

    let r3 = await ctx.service.commonSvc.commonFetch(param);

    let topics = r1
      .concat(r2)
      .concat(r3)
    ;

    topics = this.ctx.helper.betterUniqueTopics(topics);

    for (let topic of topics) {
      if (topic.pic && topic.pic.startsWith('data:image')) {
        topic.pic = undefined;
      }

      let tags = topic.tags;
      if (tags) {
        let new_tags = [];
        for (let tag of tags) {
          if (tag.includes('·')) {
            tag = tag.split('·');
            new_tags = new_tags.concat(tag);
          } else {
            new_tags.push(tag);
          }
          topic.tags = new_tags;
        }

        topic.tags = topic.tags.filter(v => !v.includes('点赞'));
      }
    }

    return topics;
  }

  async infoq(url, boards) {
    let param = {
      url: url,
      author_info: {
        author_id: '5f743b7915f22a302d318b59',
        author_name: 'InfoQ',
        author_avatar: 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/infoq.jpg',
      },
      topic_info: {
        tab: 'tech',
        boards: [],
      },
      selector_info: {
        box: '.list > .article-item',
        title: 'div.info > h6 > a',
        content: 'div.info > h6 > a',
        pic: 'div > img',
        intro: '.summary.fadeSymmary',
        visit_count: '.views-count'
      }
    };
    let topics = await this.service.commonSvc.commonFetch(param);
    return topics;
  }


  async kaifazhetoutiao() {
    const {ctx} = this;

    let param = {
      url: 'https://toutiao.io/',
      author_info: {
        author_name: '开发者头条',
        author_id: '5f743aacf9453630119b80a7',
        author_avatar: '/public/images/toutiao.png',
      },
      topic_info: {
        tab: 'tech',
        boards: ['开发'],
      },
      selector_info: {
        box: '#daily > div > div.daily > div > div',
        title: 'h3.title > a',
        content: 'h3.title > a',
        visit_count: 'a.btn.btn-default.like-button'
      }
    };

    let topics = await this.service.commonSvc.commonFetch(param);

    topics.forEach(v => {
      if (v.title.startsWith('[推荐]')) {
        v.good = true;
      }
    });

    return topics;
  }


  async choutiFaxian() {
    return [];
    const {ctx} = this;

    let param = {
      url: 'https://dig.chouti.com/recommend',
      author_info: {
        author_id: '6219cce43e0a1c1a3a2936b7',
        author_name: '抽屉新热榜',
        author_avatar: '/public/images/chouti.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: []
      },
      selector_info: {
        box: '.link-con > .link-item',
        title: 'a.link-title',
        content: 'a.link-title',
        pic: 'img.matching,img.image-item',
        pics: 'img.matching,img.image-item',
        tags: '.link-mark > a.topic-con > span.left',
        real_author_name: '.author-name',
        visit_count: '.recommend',
      },
    };

    let topics = await ctx.service.commonSvc.commonLight(param);
    this._choutiPostProcessing(topics);

    return topics;
  }


  async choutiAll() {
    return [];
    const {ctx} = this;
    let param;

    param = {
      url: 'https://dig.chouti.com',
      author_info: {
        author_id: '6219cce43e0a1c1a3a2936b7',
        author_name: '抽屉新热榜',
        author_avatar: '/public/images/chouti.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['热点']
      },
      selector_info: {
        box: '.link-con > .link-item',
        title: 'a.link-title',
        content: 'a.link-title',
        pic: 'img.matching,img.image-item',
        pics: 'img.matching,img.image-item',
        tags: '.link-mark > a.topic-con > span.left',
        visit_count: '.recommend'
      },
    };

    let r1 = await ctx.service.commonSvc.commonLight(param);


    param = {
      url: 'https://dig.chouti.com/zone/news/',
      author_info: {
        author_id: '6219cce43e0a1c1a3a2936b7',
        author_name: '抽屉新热榜',
        author_avatar: '/public/images/chouti.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['新闻']
      },
      selector_info: {
        box: '.link-con > .link-item',
        title: 'a.link-title',
        content: 'a.link-title',
        pic: 'img.matching,img.image-item',
        pics: 'img.matching,img.image-item',
        tags: '.link-mark > a.topic-con > span.left',
        visit_count: '.recommend',
      },
    };

    let r2 = await ctx.service.commonSvc.commonLight(param);

    param = {
      url: 'https://dig.chouti.com/zone/scoff',
      author_info: {
        author_id: '6219cce43e0a1c1a3a2936b7',
        author_name: '抽屉新热榜',
        author_avatar: '/public/images/chouti.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['段子']
      },
      selector_info: {
        box: '.link-con > .link-item',
        title: 'a.link-title',
        content: 'a.link-title',
        pic: 'img.matching,img.image-item',
        pics: 'img.matching,img.image-item',
        tags: '.link-mark > a.topic-con > span.left',
        real_author_name: '.author-name',
        visit_count: '.recommend',
      },
    };

    let r3 = await ctx.service.commonSvc.commonLight(param);


    param = {
      url: 'https://dig.chouti.com/zone/tec',
      author_info: {
        author_id: '6219cce43e0a1c1a3a2936b7',
        author_name: '抽屉新热榜',
        author_avatar: '/public/images/chouti.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['互联网']
      },
      selector_info: {
        box: '.link-con > .link-item',
        title: 'a.link-title',
        content: 'a.link-title',
        pic: 'img.matching,img.image-item',
        pics: 'img.matching,img.image-item',
        tags: '.link-mark > a.topic-con > span.left',
        real_author_name: '.author-name',
        visit_count: '.recommend',
      },
    };

    let r4 = await ctx.service.commonSvc.commonLight(param);


    param = {
      url: 'https://dig.chouti.com/zone/ask',
      author_info: {
        author_id: '6219cce43e0a1c1a3a2936b7',
        author_name: '抽屉新热榜',
        author_avatar: '/public/images/chouti.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['你问我答']
      },
      selector_info: {
        box: '.link-con > .link-item',
        title: 'a.link-title',
        content: 'a.link-title',
        pic: 'img.matching,img.image-item',
        pics: 'img.matching,img.image-item',
        tags: '.link-mark > a.topic-con > span.left',
        real_author_name: '.author-name',
        visit_count: '.recommend',
      },
    };

    let r5 = await ctx.service.commonSvc.commonLight(param);

    param = {
      url: 'https://dig.chouti.com/all/man',
      author_info: {
        author_id: '6219cce43e0a1c1a3a2936b7',
        author_name: '抽屉新热榜',
        author_avatar: '/public/images/chouti.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['热点']
      },
      selector_info: {
        box: '.link-con > .link-item',
        title: 'a.link-title',
        content: 'a.link-title',
        pic: 'img.matching,img.image-item',
        pics: 'img.matching,img.image-item',
        tags: '.link-mark > a.topic-con > span.left',
        real_author_name: '.author-name',
        visit_count: '.recommend',
      },
    };

    let r6 = await ctx.service.commonSvc.commonLight(param);


    param = {
      url: 'https://dig.chouti.com/zone/pic',
      author_info: {
        author_id: '6219cce43e0a1c1a3a2936b7',
        author_name: '抽屉新热榜',
        author_avatar: '/public/images/chouti.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['图片']
      },
      selector_info: {
        box: '.waterfall-main > .pic-item',
        title: 'a.link-title',
        content: 'a.link-title',
        pic: 'img.matching',
        tags: '.link-detail > .topic-con > span',
        visit_count: '.recommend',
      },
    };

    let r11 = await this.service.commonSvc.commonFetch(param);


    param = {
      url: 'https://dig.chouti.com/zone/video',
      author_info: {
        author_id: '6219cce43e0a1c1a3a2936b7',
        author_name: '抽屉新热榜',
        author_avatar: '/public/images/chouti.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['视频']
      },
      selector_info: {
        box: '.waterfall-main > .pic-item',
        title: 'a.link-title',
        content: 'a.link-title',
        pic: 'img.matching',
        tags: '.link-detail > .topic-con > span',
        visit_count: '.recommend',
      },
    };

    let r12 = await this.service.commonSvc.commonFetch(param);


    let param_huati = {
      author_info: {
        author_id: '6219cce43e0a1c1a3a2936b7',
        author_name: '抽屉新热榜',
        author_avatar: '/public/images/chouti.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: []
      },
      selector_info: {
        box: '.link-con > .link-item',
        title: 'a.link-title',
        content: 'a.link-title',
        pic: 'img.matching,img.image-item',
        pics: 'img.matching,img.image-item',
        tags: '.link-mark > a.topic-con > span.left',
        real_author_name: '.author-name',
        visit_count: '.recommend',
      },
    };


    param_huati.url = 'https://dig.chouti.com/page/section/links?sectionId=1216';
    let r_douban = await ctx.service.commonSvc.commonFetch(param_huati);

    param_huati.url = 'https://dig.chouti.com/page/section/links?sectionId=1199';
    param_huati.topic_info.boards = ['美食'];
    let r_meishi = await ctx.service.commonSvc.commonFetch(param_huati);

    param_huati.url = 'https://dig.chouti.com/page/section/links?sectionId=1219';
    param_huati.topic_info.boards = ['美食'];
    let r_meishi2 = await ctx.service.commonSvc.commonFetch(param_huati);

    param_huati.url = 'https://dig.chouti.com/page/section/links?sectionId=1286';
    param_huati.topic_info.boards = ['汽车'];
    let r_qiche = await ctx.service.commonSvc.commonFetch(param_huati);

    param_huati.url = 'https://dig.chouti.com/page/section/links?sectionId=1220';
    param_huati.topic_info.boards = ['百科'];
    let r_kepu = await ctx.service.commonSvc.commonFetch(param_huati);


    param_huati.url = 'https://dig.chouti.com/page/section/links?sectionId=1297';
    param_huati.topic_info.boards = ['脑洞'];
    let r_naodong = await ctx.service.commonSvc.commonFetch(param_huati);


    let topics = r_douban
      .concat(r_meishi)
      .concat(r_meishi2)
      .concat(r_qiche)
      .concat(r_kepu)
      .concat(r_naodong)
      .concat(r1)
      .concat(r2)
      .concat(r3)
      .concat(r4)
      .concat(r5)
      .concat(r6)
      .concat(r11)
      .concat(r12)
    ;

    topics = this.ctx.helper.betterUniqueTopics(topics);
    this._choutiPostProcessing(topics);

    return topics;
  }

  _choutiPostProcessing(topics) {
    for (let i = 0; i < topics.length; i++) {
      let topic = topics[i];
      let title = topic.title;

      let real_author_name = topic.real_author_name;
      if (real_author_name == '西城网警') {
        topics.splice(i--, 1);
        continue;
      }
      if (topic.content.includes('forgifs.com')) {
        topics.splice(i--, 1);
        continue;
      }
      if (title.includes('【') && title.includes('】')) {
        let match_result = title.match(/【(.+?)】(.+)/);
        if (match_result) {
          topic.title = match_result[1].trim();
          topic.intro = match_result[2].trim();
        }
      } else if (title.includes(' / ')) {
        topic.title = title.split(' / ')[0];
      }
      if (topic.pic && !topic.pic.includes('.gif')) {
        topic.pic = topic.pic.replace(/(.+)\?image.+/, '$1');
        topic.pic += '?imageView2/2/w/500/interlace/1';
      }
      if (topic.pics) {
        topic.pics.forEach(v => {
          v = v.replace(/(.+)\?image.+/, '$1');
          v += '?imageView2/2/w/500/interlace/1';
        });
      }

      if (topic.content) {
        if (topic.content.includes('.bilibili.com')
          || topic.content.includes('video.sina.com.cn')
          || topic.content.includes('//b23.tv')
          || topic.content.includes('video.weibo.com')
        ) {
          topic.intro = undefined;
          if (!topic.boards.includes('视频')) {
            topic.boards.push('视频');
          }
          if (!topic.tags.includes('视频')) {
            topic.tags.push('视频');
          }
        }
      }
    }


    for (const topic of topics) {
      if (topic.content.includes('sohu.com') && topic.pic && topic.pic.includes('.png')) {
        topic.pic = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora%2Fsohu_default.jpeg';
      }
    }

    // 过滤掉不需要的内容
    for (let i = 0; i < topics.length; i++) {
      let topic = topics[i];
      if (topic.content.includes('.acfun.cn')
        || topic.content.includes('.hupu.com')
        || topic.content.includes('nowre.com')
        || topic.content.includes('acg.178.com')
        || topic.content.includes('.expreview.com')
        || topic.content.includes('//bcy.net')
        || topic.content.includes('juejin.im')
        || topic.content.includes('hypebeast.cn')
        || topic.content.includes('.pixivision.net')
        || topic.content.includes('//atctwn.com')
        || topic.content.includes('//igem.ly')
        || topic.content.includes('news.ladymax.cn')
        || topic.content.includes('chongdiantou.com')
        || topic.content.includes('.ifeng.com')
        || topic.content.includes('www.xiachufang.com/dish')
        || topic.content.includes('36kr.com')
        || topic.content.includes('.ithome.com')
        || topic.content.includes('.jiemian.com')
        || topic.content.includes('.sina.')
        || topic.content.includes('.zol.com.cn')
      ) {
        topics.splice(i--, 1);
        continue;
      }

      if (topic.tags.includes('娱乐Juan')) {
        topics.splice(i--, 1);
        continue;
      }

      if (topic.content.includes('.xiachufang.com')) {
        topic.boards = topic.tags = [];
      }
    }
    this.ctx.helper.setRealAuthorName(topics);
    this.ctx.helper.setDefaultPic(topics);
    this.ctx.helper.setDefaultBoards(topics);

    for (let i = 0; i < topics.length; i++) {
      let topic = topics[i];
      if (topic.title.charCodeAt(0) == 12644) {
        topic.title = topic.title.substr(6, topic.title.length - 1);
      }
      if (topic.tags.includes('停车场')) {
        topic.tags = ['汽车'];
      } else if (topic.tags.includes('美食圈')) {
        topic.tags = ['美食'];
      } else if (topic.tags.includes('趣味科普')) {
        topic.tags = ['百科'];
      }
    }

    for (let i = 0; i < topics.length; i++) {
      let topic = topics[i];
      if (!topic.pic && topic.author_name == '抽屉新热榜') {
        topics.splice(i--, 1);
        continue;
      }
    }
  }


  async zhihuAll() {
    return [];
    const {ctx} = this;

    // let r1 = await ctx.service.crawlerSvc.zhihuTop('精选', 'total');
    // let r2 = await ctx.service.crawlerSvc.zhihuTop('科学', 'science');
    // let r3 = await ctx.service.crawlerSvc.zhihuTop('汽车', 'car');
    // let r4 = await ctx.service.crawlerSvc.zhihuTop('时尚', 'fashion');
    // let r5 = await ctx.service.crawlerSvc.zhihuTop('数码', 'digital');
    // let r6 = await ctx.service.crawlerSvc.zhihuTop('体育', 'sport');
    // let r7 = await ctx.service.crawlerSvc.zhihuTop('校园', 'school');
    // let r8 = await ctx.service.crawlerSvc.zhihuTop('影视', 'film');
    let r9 = await ctx.service.rssSvc.zhihu();

    let topics = r9;

    topics.forEach(v => {
      if (!v.pic) {
        v.pic = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/zhihu-big.jpg';
      }
    })

    return topics;
  }

  async zhihuTop(category_name, category_code) {
    const {ctx} = this;
    let totalTopics = [];

    const url = `https://www.zhihu.com/api/v3/feed/topstory/hot-lists/${category_code}?limit=40`;
    const author_id = '620899babc0e87448a243aad';
    const author_name = '知乎推荐';
    const author_avatar = '/public/images/zhihu.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    const tab = 'yutang';
    const boards = [category_name];

    const response = await got.get(url);

    let items = response.data.data;

    for (let item of items) {
      const title = item.target.title;
      const content = `https://www.zhihu.com/question/${item.target.id}`;
      const intro = item.target.excerpt;
      let pic;
      if (item.children && item.children[0]) {
        pic = item.children[0].thumbnail;
        if (pic == '') {
          pic = undefined;
        }
      }

      let topic = {
        title,
        content,
        original_url: content,
        intro,
        pic,
        tab,
        tags: boards,
        boards,
        author_id,
        author_name,
        author_avatar,
        source,
      };
      totalTopics.push(topic);
    }

    return totalTopics;
  }


  async testerhome() {
    const {ctx} = this;
    let param = {
      url: 'https://testerhome.com/topics/excellent',
      author_info: {
        author_name: '测试之家',
        author_id: '62481773991093084f30ad82',
        author_avatar: '/public/images/testerhome.png',
      },
      topic_info: {
        tab: 'tech',
        boards: ['测试'],
      },
      selector_info: {
        box: '.topic.media',
        title: '.title > a',
        content: '.title > a',
      }
    };

    let r1 = await this.service.commonSvc.commonLight(param);

    param.url = 'http://testerhome.com/topics/popular';
    let r2 = await this.service.commonSvc.commonLight(param);

    let topics = r1.concat(r2);

    topics = this.ctx.helper.betterUniqueTopics(topics);

    topics.forEach(v => {
      let tmp_arr = v.title.split('\n');
      if (tmp_arr.length > 1) {
        v.title = tmp_arr[1].trim();
        v.tags = [tmp_arr[0].trim()];
      }
    });

    for (let i = 0; i < topics.length; i++) {
      let topic = topics[i];
      if (topic.title.includes('求职')
        || topic.title.includes('社招')
        || topic.title.includes('校招')
        || topic.title.includes('招聘')
        || topic.title.includes('诚聘')
      ) {
        topics.splice(i--, 1);
        continue;
      }
    }

    return topics;
  }

  async bohaishibei() {
    const {ctx} = this;

    let param = {
      url: 'https://www.bohaishibei.com/post/category/main/',
      author_info: {
        author_id: '61e3d5d46e41fed74ac3b822',
        author_name: '博海拾贝',
        author_avatar: '/public/images/bohaishibei.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['搞笑']
      },
      selector_info: {
        box: '.content-list > div',
        title: '.entry-title > a',
        content: '.entry-title > a',
        pic: '.thumbnail-wrap > img'
      },
    };

    let topics = await ctx.service.commonSvc.commonLight(param);

    topics.forEach(v => {
      v.pic = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/bohaishibei_big.jpeg';
    })

    return topics;
  }


  async cbndata() {
    let totalTopics = [];

    const url = `https://www.cbndata.com/information`;
    const tag = '看点';

    const tab = 'yutang';
    const source = 'CBNData' + dayjs().format('YYYY-MM-DD');
    const author_name = 'CBNData';
    const author_id = '6219e6fade9f522069f648c0';
    const author_avatar = '/public/images/cbndata.png';

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.slick-slide');
      const eles_top = await page.$$('.slick-slide');
      const eles = await page.$$('.cp-information-list-inner-wrapper > li');

      for (const el of eles_top) {
        if (!await el.$('.item-info-title')) {
          continue;
        }
        let title = await el.$eval('.item-info-title', v => v.innerText);
        let content = await el.$eval('a', v => v.href);
        let intro = await el.$eval('.item-info-summary', v => v.innerText);
        let pic = await el.$eval('a> img', v => v.src);
        let tags = [tag];

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          intro,
          tags,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }

    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async cbndataReport() {
    let totalTopics = [];

    const url = `https://www.cbndata.com/report`;
    const tag = '报告';

    const tab = 'yutang';
    const source = 'CBNData' + dayjs().format('YYYY-MM-DD');
    const author_name = 'CBNData';
    const author_id = '6219e6fade9f522069f648c0';
    const author_avatar = '/public/images/cbndata.png';

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.basic-report');
      const eles = await page.$$('.basic-report');

      await this.ctx.helper.autoScrollByLimit(page, 5000);

      for (const el of eles) {
        let title = await el.$eval('.report-title.cursor-pointer', v => v.innerText);
        let content = await el.$eval('.report-download.cursor-pointer', v => v.getAttribute('data-id'));
        content = `https://www.cbndata.com/report/${content}/detail?isReading=report&page=1`;
        let pic = await el.$eval('.fixed-ratio-image > img', v => v.src);
        let tags = await el.$$eval('.cp-tag-unit > a', vs => vs.map(v => v.innerText.replace('#', '')));
        tags.push('报告');

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          tags,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }


    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }

  async ctrip() {
    let totalTopics = [];

    const url = 'https://you.ctrip.com/travels/fall/t2.html';

    const tab = 'yutang';
    const author_name = '携程游记';
    const author_id = '620b8e77bb677b4c599e5c1a';
    const author_avatar = '/public/images/ctrip.png';
    const source = '携程游记' + dayjs().format('YYYY-MM-DD');

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});
      await page.waitForSelector('.forfall .city');
      const eles = await page.$$('.forfall .city');

      for (const el of eles) {
        const title = await el.$eval('.cpt', v => v.innerText);
        const content = await el.$eval('.cpt', v => v.href);
        let pic;
        if (await el.$('a.city-image > img.pic')) {
          pic = await el.$eval('a.city-image > img.pic', v => v.src);
          pic = pic.replace('_R_228_10000_Q90', '');

          let topic = {
            title,
            content,
            original_url: content,
            pic,
            tab,
            source,
            author_id,
            author_name,
            author_avatar,
          };
          totalTopics.push(topic);
        }
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error('携程游记爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async dongqiudizaobao() {
    let zaobao;
    const url = 'https://www.dongqiudi.com/special/48/';
    const tag = '早报';

    const tab = 'yutang';
    const author_name = '懂球帝';
    const author_id = '6217903c0ca6b665eb94c721';
    const author_avatar = '/public/images/dongqiudi.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    let browser;
    try {
      // 启动浏览器
      browser = await chromium.launch({headless: true});
      const context = await browser.newContext();

      // 打开新的页面
      const page = await context.newPage();

      // 导航到指定的 URL
      await page.goto('https://www.dongqiudi.com/special/48/');

      // 等待指定的元素加载
      const selector = '#__layout > div > div:nth-child(2) > div.special-father > div:nth-child(2) > div:nth-child(1) > div';
      await page.waitForSelector(selector);

      // 点击元素并等待新的页面打开
      const [newPage] = await Promise.all([
        context.waitForEvent('page'), // 等待新页面事件
        page.click(selector) // 点击元素
      ]);

      // 等待新页面加载
      await newPage.waitForLoadState();

      // 获取并输出新页面的 URL
      console.log('New page URL:', newPage.url());
      const title = await newPage.title();
      const content = newPage.url();

      zaobao = {
        title,
        content,
        original_url: content,
        tags: [tag],
        tab,
        author_id,
        author_name,
        author_avatar,
        source,
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      // 关闭浏览器
      await browser.close();
      return zaobao;
    }
  }


  async fengwen() {
    let param = {
      url: 'https://user.guancha.cn',
      author_info: {
        author_name: '观察者网',
        author_id: '6215027814ead414b2000280',
        author_avatar: '/public/images/guancha.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['风闻'],
        tags: ['风闻'],
      },
      selector_info: {
        box: 'li.index-list-item',
        title: '.list-item > h4 > a',
        content: '.list-item > h4 > a',
        pic: '.item-pic img',
        intro: '.item-content',
        visit_count: '.comment'
      },
    };

    let topics = await this.service.commonSvc.commonLight(param);

    topics = await this.service.topicSvc.filterExistTopics(topics);

    for (let i = 0; i < topics.length; i++) {
      let topic = topics[i];
      if (!topic.pic) {
        topics.splice(i--, 1);
      }
    }

    let result = await Promise.all(
      topics.map(async (item) => {
        const detailResponse = await got.get(item.content);

        const $ = cheerio.load(detailResponse.data);

        let tags = $('.hot-topic-nav:first-of-type > a').map((i, el) => {
          return $(el).text();
        }).get();

        item.tags = item.tags.concat(tags);

        return item;
      })
    );

    // result.forEach(v => {
    //   if (!v.pic) {
    //     v.pic = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/fengwen-big.png';
    //   }
    // })


    return result;
  }


  async guojiadili(url, board, tag) {
    let totalTopics = [];

    const tab = 'yutang';

    const author_id = '621a1f1de4fef3247f2949eb';
    const author_name = '国家地理';
    const author_avatar = '/public/images/guojiadili.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    try {
      const response = await got({url: url, method: 'get'});

      const $ = cheerio.load(response.data);

      // 左边大块
      let topic_big = {
        title: $('div.artcle-btn-large > div > div > div > h4 > a.text-white').text().trim(),
        content: $('div.artcle-btn-large > div > div > div > h4 > a.text-white').attr('href'),
        pic: $('div.artcle-btn-large > div > div > a > div > img').attr('src')
      }
      totalTopics.push(topic_big);

      // 下边大块2
      let topic_big2 = {
        title: $('body > section.content-all > div > div.article-link-w65 > div.art-w65-left > div.art-btn-la-content > div > h4 > a.text-white').text().trim(),
        content: $('body > section.content-all > div > div.article-link-w65 > div.art-w65-left > div.art-btn-la-content > div > h4 > a.text-white').attr('href'),
        pic: $('body > section.content-all > div > div.article-link-w65 > div.art-w65-left > div.art-btn-la-content > a > div > img').attr('src')
      }
      totalTopics.push(topic_big2);

      // 右边三小块
      let items_right = $('.artile-btn-small .art-btn-s-content');
      for (const el of items_right) {
        let topic = {
          title: $(el).find('h4 > a.text-black').text().trim(),
          content: $(el).find('h4 > a.text-black').attr('href'),
          pic: $(el).find('div.art-btn-s-img img').attr('src')
        }
        totalTopics.push(topic);
      }


      // 中间三小块
      let items_middle = $('.artcle-btn-mv');
      for (const el of items_middle) {
        let topic = {
          title: $(el).find('h4 > a.text-black').text().trim(),
          content: $(el).find('h4 > a.text-black').attr('href'),
          pic: $(el).find('div.art-btn-mv-img img').attr('src')
        }
        totalTopics.push(topic);
      }

      // 下边长列表
      let items_bottom = $('div.art-w65-left > div.artcle-btn-mh');
      for (const el of items_bottom) {
        let topic = {
          title: $(el).find('h4 > a.text-black').text().trim(),
          content: $(el).find('h4 > a.text-black').attr('href'),
          pic: $(el).find('div.art-btn-mh-img img').attr('src')
        }
        if (topic.title) {
          totalTopics.push(topic);
        }
      }


      for (const topic of totalTopics) {
        topic.title = chinese.traditionToSimple(topic.title);
        topic.original_url = topic.content;
        topic.boards = [board];
        topic.tags = [board, tag, '百科'];
        topic.tab = tab;
        topic.author_id = author_id;
        topic.author_name = author_name;
        topic.author_avatar = author_avatar;
        topic.source = source;
      }

    } catch (e) {
      this.logger.error(e);
    } finally {
      if (totalTopics.length == 0) {
        this.logger.error(author_name, board, tag, '爬取异常!!!');
      } else {
        this.logger.info(author_name, board, tag, 'success...数量为:', totalTopics.length);
      }
      return totalTopics;
    }
  }


  async guojiadiliVideo(url, board) {
    let totalTopics = [];

    const tab = 'yutang';
    const tag = '影音精选';

    const author_id = '621a1f1de4fef3247f2949eb';
    const author_name = '国家地理';
    const author_avatar = '/public/images/guojiadili.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    try {
      const response = await got({url: url, method: 'get'});

      const $ = cheerio.load(response.data);

      // 上边大块
      let topic_big = {
        title: $('body > section.content-all > div > div.video-link-w100 > div.video-btn-xlarge > div > div > h4 > a.text-white').text().trim(),
        content: $('body > section.content-all > div > div.video-link-w100 > div.video-btn-xlarge > div > div > h4 > a.text-white').attr('href'),
        pic: $('body > section.content-all > div > div.video-link-w100 > div.video-btn-xlarge > div > a > div > img').attr('src')
      }
      totalTopics.push(topic_big);


      // 下边两大块
      let items_right = $('.video-btn-large');
      for (const el of items_right) {
        let topic = {
          title: $(el).find('h4 > a.text-white').text().trim(),
          content: $(el).find('h4 > a.text-white').attr('href'),
          pic: $(el).find('div.video-btn-la-img img').attr('src')
        }
        totalTopics.push(topic);
      }


      // 上边四小块
      let items_middle = $('.video-btn-small');
      for (const el of items_middle) {
        let topic = {
          title: $(el).find('h4 > a.text-black').text().trim(),
          content: $(el).find('h4 > a.text-black').attr('href'),
          pic: $(el).find('div.video-btn-img img').attr('src')
        }
        totalTopics.push(topic);
      }

      // 下边八小块
      let items_bottom = $('.video-btn-middle');
      for (const el of items_bottom) {
        let topic = {
          title: $(el).find('h4 > a.text-black').text().trim(),
          content: $(el).find('h4 > a.text-black').attr('href'),
          pic: $(el).find('div.video-btn-img img').attr('src')
        }
        if (topic.title) {
          totalTopics.push(topic);
        }
      }


      for (const topic of totalTopics) {
        topic.title = chinese.traditionToSimple(topic.title);
        topic.original_url = topic.content;
        topic.boards = [board];
        topic.tags = [board, tag, '百科'];
        topic.tab = tab;
        topic.author_id = author_id;
        topic.author_name = author_name;
        topic.author_avatar = author_avatar;
        topic.source = source;
      }

    } catch (e) {
      this.logger.error(e);
    } finally {
      if (totalTopics.length == 0) {
        this.logger.error(author_name, board, tag, '爬取异常!!!');
      } else {
        this.logger.info(author_name, board, tag, 'success...数量为:', totalTopics.length);
      }
      return totalTopics;
    }
  }

  async guokrAll() {
    // 果壳
    let url = `https://www.guokr.com/apis/minisite/article.json?retrieve_type=by_wx&channel_key=predator&offset=0&limit=20`;
    let boards = ['吃货研究所'];
    let g1 = await this.ctx.service.crawlerSvc.guokr(url, boards);

    url = `https://www.guokr.com/apis/minisite/article.json?retrieve_type=by_wx&channel_key=pac&offset=0&limit=20`;
    boards = ['物种日历'];
    let g2 = await this.ctx.service.crawlerSvc.guokr(url, boards);

    url = `https://www.guokr.com/beta/proxy/science_api/articles?retrieve_type=by_category&page=1`;
    boards = ['科学人'];
    let g3 = await this.ctx.service.crawlerSvc.guokr(url, boards);

    let topics = g1.concat(g2).concat(g3);
    topics = this.ctx.helper.betterUniqueTopics(topics);

    for (const topic of topics) {
      if (topic.pic) {
        topic.pic = topic.pic.replace(/(.+)\?image.+/, '$1');
        topic.pic += '?imageView2/2/w/500/interlace/1';
      }
    }

    return topics;
  }

  async guokr(url, boards) {
    let totalTopics = [];

    const tab = 'yutang';
    const source = '果壳网' + dayjs().format('YYYY-MM-DD');
    const author_name = '果壳网';
    const author_id = '61dda2a097ceba9ea3ea8ca2';
    const author_avatar = '/public/images/guokr.png';

    try {
      let response = await got.get(url);

      let objs = response.data;

      if (objs.result) {
        objs = objs.result;
      }
      for (let obj of objs) {
        let topic = {
          title: obj.title,
          content: `https://www.guokr.com/article/${obj.id}`,
          intro: obj.summary,
          pic: obj.small_image,
          original_url: `https://www.guokr.com/article/${obj.id}`,
          boards: ['百科', ...boards],
          tags: ['百科', ...boards],
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };
        totalTopics.push(topic);
      }

    } catch (e) {
      this.logger.error(e);
    } finally {
      // await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error('果壳网', '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async historyTody() {
    const browser = await chromium.launch({headless: true});

    let totalTopics = [];
    let url = `https://hao.360.com/histoday/`;

    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.tih-item');

      const eles = await page.$$('.tih-item');

      for (const el of eles) {
        await el.click();
        await page.waitForTimeout(500);
        let title = await el.$eval('dt', v => v.innerText);
        let original_url = await el.$eval('.read-btn', v => v.href);
        let intro = await el.$eval('.desc', v => v.innerText);
        let pic;
        if (await el.$('.item-img')) {
          pic = await el.$eval('.item-img', v => v.src);
        }
        let topic = {
          title: title.trim(),
          original_url: original_url.trim(),
          intro: intro.trim(),
          pic
        };
        totalTopics.push(topic);
      }

    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error('历史上的今天', '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async infoqWeekly() {
    let totalTopics = [];

    const url = 'https://www.infoq.cn/weekly/landing';

    const tab = 'tech';
    const author_id = '5f743b7915f22a302d318b59';
    const author_name = 'InfoQ';
    const author_avatar = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/infoq.jpg';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    const browser = await chromium.launch({headless: true});
    try {
      const context = await browser.newContext();
      const page = await context.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.weekly-title.new');


      let newPage;
      [newPage] = await Promise.all([
        context.waitForEvent('page', {timeout: 10000}),
        page.click('.weekly-title.new'),
      ]);


      await newPage.waitForSelector('body > table > tbody > tr > td > table > tbody > tr:nth-child(3) > td > table:nth-child(2) > tbody > tr > td > table > tbody > tr');
      const eles = await newPage.$$('body > table > tbody > tr > td > table > tbody > tr:nth-child(3) > td > table:nth-child(2) > tbody > tr > td > table > tbody > tr');

      console.log(eles.length);


      let top_title = await newPage.$eval('body > table > tbody > tr > td > table > tbody > tr:nth-child(3) > td > table:nth-child(1) > tbody > tr > td > table > tbody > tr:nth-child(3) > td > table > tbody > tr:nth-child(2) > td > p > a', v => v.innerText);
      let top_content = await newPage.$eval('body > table > tbody > tr > td > table > tbody > tr:nth-child(3) > td > table:nth-child(1) > tbody > tr > td > table > tbody > tr:nth-child(3) > td > table > tbody > tr:nth-child(2) > td > p > a', v => v.href);
      let top_pic = await newPage.$eval('body > table > tbody > tr > td > table > tbody > tr:nth-child(3) > td > table:nth-child(1) > tbody > tr > td > table > tbody > tr:nth-child(3) > td > table > tbody > tr:nth-child(1) > td > img', v => v.src);

      let top_topic = {
        title: top_title,
        content: top_content,
        original_url: top_content,
        pic: top_pic,
        tab,
        boards: ['技术周刊'],
        tags: ['技术周刊'],
        source,
        author_id,
        author_name,
        author_avatar,
      };

      totalTopics.push(top_topic);


      for (const el of eles) {
        let title;
        let content;
        let pic;
        let intro;
        if (!await el.$('a')) {
          continue;
        }
        title = await el.$eval('a', v => v.innerText);
        content = await el.$eval('a', v => v.href);

        if (await el.$('img')) {
          pic = await el.$eval('img', v => v.src);
        }
        if (await el.$('td > table > tbody > tr > td:nth-child(1) > table > tbody > tr:nth-child(2) > td > p > a')) {
          intro = await el.$eval('td > table > tbody > tr > td:nth-child(1) > table > tbody > tr:nth-child(2) > td > p > a', v => v.innerText);
        }

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          intro,
          tab,
          boards: ['技术周刊'],
          tags: ['技术周刊'],
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async jiandan() {
    let param = {
      url: 'http://jandan.net/',
      author_info: {
        author_id: '61fe8a77ea08a323866eba99',
        author_name: '煎蛋网',
        author_avatar: '/public/images/jiandan.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['百科'],
      },
      selector_info: {
        box: '.post.f.list-post',
        title: 'div.indexs > h2 > a',
        content: 'div.indexs > h2 > a',
        pic: '.thumbs img',
        tags: 'strong > a',
        intro: 'div.indexs'
      },
      scroll: 3000
    };

    let topics = await this.service.commonSvc.commonFetch(param);

    for (let i = 0; i < topics.length; i++) {
      let topic = topics[i];
      if (topic.title.includes('好价')) {
        topics.splice(i--, 1);
        continue;
      }
      if (topic.pic == 'https://pic.yupoo.com/jdvip/BQSJzdk5/custom.jpg') {
        topic.pic = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/jiandan-default.jpg';
        topic.pic = topic.pic.replace('!custom', '');
      }
      if (topic.intro) {
        topic.intro = topic.intro.split('\n');
        topic.intro = topic.intro[topic.intro.length - 1];
      }
    }
    return topics;
  }


  async jiemianAccount() {
    let totalTopics = [];

    const url = 'https://www.jiemian.com/account/caijing.html';
    const boards = ['界面号'];
    const tags = ['界面号'];

    const tab = 'yutang';
    const source = '界面新闻' + dayjs().format('YYYY-MM-DD');
    const author_name = '界面新闻';
    const author_id = '6219cae43a07421946682580';
    const author_avatar = '/public/images/jiemian.png';

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.news-view.card');
      const eles = await page.$$('.news-view.card');

      for (const el of eles) {
        if (!await el.$('.news-img > a > img')) {
          continue;
        }
        let title = await el.$eval('h3 > a', v => v.innerText);
        let content = await el.$eval('h3 > a', v => v.href);
        let pic = await el.$eval('.news-img > a > img', v => v.src);

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          boards,
          tags,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async jiqizhixin() {
    let totalTopics = [];

    const url = `https://www.jiqizhixin.com/`;

    const tab = 'yutang';
    const source = '机器之心' + dayjs().format('YYYY-MM-DD');
    const author_name = '机器之心';
    const author_id = '6219e582ef67211fab05043b';
    const author_avatar = '/public/images/jiqizhixin.png';

    let browser;
    let page;
    try {
      [browser, page] = await this.ctx.helper.getCanUseBrowserAndPage(url, '.article-item__container');
      const eles = await page.$$('.article-item__container');

      for (const el of eles) {
        let title = await el.$eval('a.article-item__title', v => v.innerText);
        let content = await el.$eval('a.article-item__title', v => v.href);
        let intro = await el.$eval('p.article-item__summary', v => v.innerText);
        let pic = await el.$eval('img.u-image-center', v => v.src);
        let tags = await el.$$eval('.u-btn--gray.category__link', vs => vs.map(v => v.innerText));

        let topic = {
          title,
          content,
          original_url: content,
          intro,
          pic,
          tags,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      if (browser) {
        await browser.close();
      }
      if (totalTopics.length == 0) {
        this.logger.error('机器之心爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async juejin(url, boards) {
    let totalTopics = [];

    const tab = 'tech';
    const source = '掘金' + dayjs().format('YYYY-MM-DD');
    const author_id = '5f743ad00c1ddb3019f42456';
    const author_name = '掘金';
    const author_avatar = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/juejin.webp';

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.entry-list > .item', {state: 'attached'});
      const eles = await page.$$('.entry-list > .item');

      await this.ctx.helper.autoScrollByLimit(page, 3000);

      for (const el of eles) {
        let title = await el.$eval('.title', v => v.innerText);
        let content = await el.$eval('.title', v => v.href);

        let intro;
        if (await el.$('.abstract')) {
          intro = await el.$eval('.abstract', v => v.innerText);
        } else if (await el.$('.description')) {
          intro = await el.$eval('.description', v => v.innerText);
        }

        let pic;
        if (await el.$('img.lazy.thumb')) {
          pic = await el.$eval('img.lazy.thumb', v => v.src);
          if (pic.startsWith('data:image')) {
            pic = undefined;
          }
          pic = pic.replace(':108:72:0:0:q75.avis', ':216:144:0:0:q75.awebp');
        }

        let visit_count = 0;
        if (await el.$('.item.view')) {
          visit_count = await el.$eval('.item.view', v => v.innerText);
          visit_count = visit_count.trim();
          if (visit_count.endsWith('k')) {
            visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 1000;
          } else if (visit_count.endsWith('w')) {
            visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 10000;
          } else {
            visit_count = +visit_count;
          }
        }


        let tags = [];
        if (await el.$('.tag')) {
          let tmp_tags = await el.$$eval('.tag', vs => vs.map(v => v.innerText));
          tags = tags.concat(tmp_tags);
        }

        let is_adv = false;
        if (tags) {
          if (JSON.stringify(tags).includes('广告')) {
            is_adv = true;
          }
        }

        if (!is_adv) {
          let topic = {
            title,
            content,
            original_url: content,
            pic,
            intro,
            visit_count,
            boards: [...boards],
            tags,
            tab,
            source,
            author_id,
            author_name,
            author_avatar,
          };
          totalTopics.push(topic);
        }
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error('掘金爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async kankan() {
    let totalTopics = [];

    const url = `https://www.kankanews.com/`;

    const tab = 'yutang';
    const source = '看看新闻' + dayjs().format('YYYY-MM-DD');
    const author_id = '621a27f5d4259428c0896f47';
    const author_name = '看看新闻';
    const author_avatar = '/public/images/kankanxinwen.png';

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.topline.left.clearfix');
      const eles_top = await page.$$('.topline.left.clearfix');

      const eles = await page.$$('ul.clearfix.conlist.conlistindex > li');

      for (const el of eles_top) {
        let title = await el.$eval('.right.infor > a', v => v.innerText);
        let content = await el.$eval('.right.infor > a', v => v.href);
        let intro = await el.$eval('p', v => v.innerText);
        let pic = await el.$eval('.left.pic img', v => v.src);

        let topic = {
          title,
          content,
          original_url: content,
          intro,
          pic,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };
        totalTopics.push(topic);
      }

      for (const el of eles) {
        let title = await el.$eval('.infor > a', v => v.innerText);
        let content = await el.$eval('.infor > a', v => v.href);
        let intro = await el.$eval('p', v => v.innerText);
        let pic = await el.$eval('.pic img', v => v.src);

        let topic = {
          title,
          content,
          original_url: content,
          intro,
          pic,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };
        totalTopics.push(topic);
      }

    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async kepuchinaPiyao() {
    let totalTopics = [];

    let url = 'https://piyao.kepuchina.cn/rumor/rumorlist?type=0';

    const tab = 'yutang';
    const source = '科普中国' + dayjs().format('YYYY-MM-DD');
    const author_name = '科普中国';
    const author_id = '621a25057fc7f3275962e4ef';
    const author_avatar = '/public/images/kepuchina.png';

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.rumor-list_item.pull-left');
      let eles = await page.$$('.rumor-list_item.pull-left');

      for (const el of eles) {
        let title = await el.$eval('.rumor-list_item-title.text-ellipsis2', v => v.innerText);
        let content = await el.$eval('.pull-right > a', v => v.href);
        let pic = await el.$eval('img.rumor-list_item-img', v => v.src);

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          boards: ['辟谣'],
          tags: ['辟谣', '百科'],
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }

    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async kepuchinaTab(url, board) {
    let totalTopics = [];

    const tab = 'yutang';
    const source = '科普中国' + dayjs().format('YYYY-MM-DD');
    const author_name = '科普中国';
    const author_id = '621a25057fc7f3275962e4ef';
    const author_avatar = '/public/images/kepuchina.png';

    let browser;
    let page;
    try {
      [browser, page] = await this.ctx.helper.getCanUseBrowserAndPage(url, '.act_item.clearfix');
      const eles = await page.$$('.act_item.clearfix');
      if (eles.length == 0) {
        this.logger.error('前沿爬取异常');
      }

      for (const el of eles) {
        let title = await el.$eval('.il_text > a', v => v.innerText);
        let content = await el.$eval('.il_text > a', v => v.href);
        let pic = await el.$eval('a > img', v => v.src);
        let visit_count = await el.$eval('.il_label > span:nth-child(2)', v => v.innerText);
        visit_count = +(visit_count.trim().replace(/[^\d]/g, ''));

        let boards = [board];
        let tags = [board, '百科'];

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          visit_count,
          boards,
          tags,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      if (browser) {
        await browser.close();
      }
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async kuaikeji() {
    let totalTopics = [];

    const tab = 'yutang';

    const url = 'https://news.mydrivers.com/';
    const source = '快科技' + dayjs().format('YYYY-MM-DD');
    const author_name = '快科技';
    const author_id = '6214fe67b9c5ca132f318a5a';
    const author_avatar = '/public/images/kuaikeji.png';

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.news_lb > li');
      const eles = await page.$$('.news_lb > li');

      await this.ctx.helper.autoScrollByLimit(page, 5000);
      await page.waitForTimeout(3000);

      for (const el of eles) {
        if (!await el.$('h3 > a')) {
          continue;
        }
        if (!await el.$('p')) {
          continue;
        }

        let title = await el.$eval('h3 > a', v => v.innerText);
        let content = await el.$eval('h3 > a', v => v.href);
        let pic = await el.$eval('img.lazy', v => v.src);
        if (pic == 'https://icons.mydrivers.com/news/2018/load.gif') {
          pic = undefined;
        }
        let intro = await el.$eval('p', v => v.innerText);

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          intro,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        if (pic) {
          totalTopics.push(topic);
        }
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }

  async liangziwei() {
    let totalTopics = [];

    const tab = 'yutang';

    let url = 'https://www.qbitai.com/';
    const author_name = '量子位';
    const author_id = '6219cfac8d51f01b8aecae74';
    const author_avatar = '/public/images/liangziwei.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    try {
      let response = await got.get(url);

      let $ = cheerio.load(response.data);

      // 下边
      let eles_bottom = $('.article_list > .picture_text');
      for (const el of eles_bottom) {
        let topic = {
          title: $(el).find('h4 > a').text().trim(),
          content: $(el).find('h4 > a').attr('href'),
          pic: $(el).find('img').attr('src'),
          intro: $(el).find('p:nth-of-type(2)').text().trim(),
          tags: $(el).find('a[rel="tag"]').map((i, el) => {
            return $(el).text();
          }).get()
        }
        totalTopics.push(topic);
      }

      // 左边
      let eles_left = $('.swiper-wrapper > .swiper-slide');
      for (const el of eles_left) {
        let topic = {
          title: $(el).find('h3').text().trim(),
          content: $(el).find('a').attr('href'),
          pic: $(el).find('img').attr('src'),
        }
        totalTopics.push(topic);
      }


      url = 'https://www.qbitai.com/category/资讯'
      response = await got({
        method: 'get',
        url: url,
      });

      $ = cheerio.load(response.data);

      // 下边
      eles_bottom = $('.article_list > .picture_text');
      for (const el of eles_bottom) {
        let topic = {
          title: $(el).find('h4 > a').text().trim(),
          content: $(el).find('h4 > a').attr('href'),
          pic: $(el).find('img').attr('src'),
          intro: $(el).find('p:nth-of-type(2)').text().trim(),
          tags: $(el).find('a[rel="tag"]').map((i, el) => {
            return $(el).text();
          }).get()
        }
        totalTopics.push(topic);
      }


      for (const topic of totalTopics) {
        topic.original_url = topic.content;
        topic.tab = tab;
        topic.author_id = author_id;
        topic.author_name = author_name;
        topic.author_avatar = author_avatar;
        topic.source = source;
      }

    } catch (e) {
      this.logger.error(e);
    } finally {
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async mafengwo() {
    let totalTopics = [];

    const url = 'https://www.mafengwo.cn/';

    const tab = 'yutang';
    const author_id = '620b78dab551e146e725a837';
    const author_name = '马蜂窝';
    const author_avatar = '/public/images/mafengwo.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    const browser = await firefox.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForTimeout(7000);
      await page.click('._j_gs_tab.tn-nav-new');
      await page.waitForTimeout(7000);
      await page.waitForSelector('.tn-item');
      await this.ctx.helper.autoScrollByLimit(page, 5000);
      const eles = await page.$$('.tn-item');

      const top_eles = await page.$$('.show-image > li');

      for (const el of top_eles) {
        let title = await el.$eval('h3', v => v.innerText);
        let content = await el.$eval('a', v => v.href);
        let pic = await el.$eval('img', v => v.src);

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          boards: ['旅游'],
          tags: ['旅游'],
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }


      for (const el of eles) {
        let title = await el.$eval('div.tn-wrapper > dl > dt > a:last-of-type', v => v.innerText);
        let content = await el.$eval('div.tn-wrapper > dl > dt > a:last-of-type', v => v.href);
        let intro = await el.$eval('div.tn-wrapper > dl > dd > a', v => v.innerText);
        let pic = await el.$eval('div.tn-image > a > img', v => v.src);

        let topic = {
          title,
          content,
          original_url: content,
          intro,
          pic,
          boards: ['旅游'],
          tags: ['旅游'],
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error('马蜂窝游记爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async oschina(url, boards) {
    let totalTopics = [];

    const tab = 'tech';
    const source = '开源中国' + dayjs().format('YYYY-MM-DD');
    const author_name = '开源中国';
    const author_id = '5f74394caa01a12fdd91bce9';
    const author_avatar = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/oschina.png';

    let browser;
    let page;
    try {
      [browser, page] = await this.ctx.helper.getCanUseBrowserAndPage(url, '.blog-list-container > .item.blog-item:not(.ad-wrap)');
      const eles = await page.$$('.blog-list-container > .item.blog-item:not(.ad-wrap)');

      for (let i = 0; i < eles.length; i++) {
        let el = eles[i];
        let title = await el.$eval('a.header', v => v.innerHTML);
        title = title.replace('<div class="ui teal label horizontal" data-tooltip="原创">原</div>', '')
          .replace('<div class="ui orange label horizontal" data-tooltip="推荐">荐</div>', '')
          .replace('<div class="ui brown label horizontal" data-tooltip="转载">转</div>', '').trim();
        let content = await el.$eval('a.header', v => v.href);
        let pic;
        if (await el.$('a.ui.small.image > img')) {
          pic = await el.$eval('a.ui.small.image > img', v => v.src);
        }
        let intro;
        if (await el.$('.description > p.line-clamp')) {
          intro = await el.$eval('.description > p.line-clamp', v => v.innerText);
          intro = intro.trim().replace(/!\[\]\(.+?\)/g, '')
            .replace(/ \[(.+?)\]\(.+?\) /g, '$1')
            .replace(/\*.+?\*/, '')
            .replace(/ \[(.+?)\]/g, '$1')
            .replace(/\([^)]+$/, '').trim();
        }

        let visit_count = 0;
        if (await el.$('.item:nth-child(3)')) {
          visit_count = await el.$eval('.item:nth-child(3)', v => v.innerText);
          visit_count = visit_count.trim();
          if (visit_count.endsWith('K')) {
            visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 1000;
          } else if (visit_count.endsWith('W')) {
            visit_count = +(visit_count.replace(/[^\d.]/g, '')) * 10000;
          } else {
            visit_count = +visit_count;
          }
        }


        let topic = {
          title,
          content,
          original_url: content,
          pic,
          intro,
          visit_count,
          boards: [...boards],
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      if (browser) {
        await browser.close();
      }
      if (totalTopics.length == 0) {
        this.logger.error(author_name, boards, '爬取异常!!!');
      } else {
        this.logger.info(author_name, boards, '爬取到:', totalTopics.length);
      }

      return totalTopics;
    }
  }


  async pingwest() {
    let totalTopics = [];

    const url = `https://www.pingwest.com/`;

    const tab = 'yutang';
    const source = '品玩热榜' + dayjs().format('YYYY-MM-DD');
    const author_name = '品玩热榜';
    const author_id = '621a269f96fb082834a7853d';
    const author_avatar = '/public/images/pingwan.png';

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('article.item');
      const eles = await page.$$('article.item');

      for (const el of eles) {
        if (!await el.$('.title > a')) {
          continue;
        }
        let title = await el.$eval('.title > a', v => v.innerText);
        let content = await el.$eval('.title > a', v => v.href);
        let intro = await el.$eval('p.desc', v => v.innerText);
        let pic = await el.$eval('.news-img img', v => v.src);
        let tags = await el.$$eval('.tag', vs => vs.map(v => v.innerText));

        let topic = {
          title,
          content,
          original_url: content,
          intro,
          pic,
          tags,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error('品玩爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async qunar() {
    return [];
    const {ctx} = this;

    let param = {
      url: 'http://travel.qunar.com/',
      author_info: {
        author_id: '620bb08439c54a32c22df5f2',
        author_name: '去哪儿',
        author_avatar: '/public/images/qunar.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['旅游']
      },
      selector_info: {
        box: '.note_li',
        title: 'h2.lf > .c33',
        content: 'h2.lf > .c33',
        pic: 'ul.imgul > li.imgli > a > img',
        intro: 'dl.cont_dl.c66.clrfix:last-of-type'
      },
      scroll: 2000
    };

    let topics = await ctx.service.commonSvc.commonFetch(param);

    topics.forEach(v => {
      if (v.intro) {
        v.intro = v.intro.replace(/\n/g, '').replace(/>/g, ' > ').replace('行程：', '');
      }
      if (v.pic) {
        v.pic = v.pic.replace('200x150x95', '400x265x80');
      }
    })

    return topics;
  }


  async segmentfaultWeekly() {
    let totalTopics = [];

    const url = 'https://segmentfault.com/weekly';

    const tab = 'tech';
    const boards = ['技术周刊'];
    const author_name = '思否周刊';
    const author_id = '5f743a5cea022430046c0d7d';
    const author_avatar = '/public/images/segmentfault.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    const browser = await chromium.launch({headless: true});
    try {
      const context = await browser.newContext();
      const page = await context.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('body > div.container > ul > li:nth-child(1) > a');

      await page.click('body > div.container > ul > li:nth-child(1) > a');

      await page.waitForSelector('body > div.edm__main-warp > div > div.edm__content > div:nth-child(2) > div.edm__content-list > div');
      const eles = await page.$$('body > div.edm__main-warp > div > div.edm__content > div:nth-child(2) > div.edm__content-list > div');


      for (const el of eles) {
        let title = await el.$eval('a', v => v.innerText);
        let content = await el.$eval('a', v => v.href);

        let topic = {
          title,
          content,
          original_url: content,
          tab,
          boards,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async shaoshupai() {
    let totalTopics = [];

    const url = 'https://sspai.com/';

    const author_name = '少数派';
    const author_id = '6219cf8d1c6fd71b6d972775';
    const author_avatar = '/public/images/shaoshupai.png';
    const tab = 'yutang';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('.articleCard-box > div');
      await this.ctx.helper.autoScrollByLimit(page, 5000);
      const eles = await page.$$('.articleCard-box > div');
      const zaobaos = await page.$$('.pai.articleCard.home.pc');

      // 派早报
      for (const el of zaobaos) {
        let title;
        let content;
        let pic;
        let boards = [];
        let tags = [];

        title = await el.$$eval('.content.text_ellipsis2', vs => vs.map(v => v.innerText));
        title = `派早报：${title[0]}、${title[1]}等`;
        content = await el.$eval('.pai_title > a', v => v.href);
        pic = 'https://cdn.sspai.com/article/a1b069c0-c837-5825-f003-3d763d98e603.png?imageView2/2/w/500/interlace/1';

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          boards,
          tags,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);

      }

      for (const el of eles) {
        let title;
        let content;
        let pic;
        if (!await el.$('.card_content a')) {
          continue;
        }
        title = await el.$eval('.card_content a', v => v.innerText);
        content = await el.$eval('.card_content a', v => v.href);
        if (await el.$('.img_box img')) {
          pic = await el.$eval('.img_box img', v => v.src);
        }

        let visit_count = 0;
        if (await el.$('div.right > div.pic_box')) {
          visit_count = await el.$eval('div.right > div.pic_box', v => v.innerText);
          visit_count = +(visit_count.trim().replace(/[^\d]/g, ''));
        }


        let topic = {
          title,
          content,
          original_url: content,
          pic,
          visit_count,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }

      for (const topic of totalTopics) {
        if (topic.pic) {
          topic.pic = topic.pic.replace(/(.+)\?image.+/, '$1');
          topic.pic += '?imageView2/2/w/500/interlace/1';
        }
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async shejipi() {
    let totalTopics = [];

    const tab = 'yutang';

    const url = 'https://www.shejipi.com/';
    const author_name = '设计癖';
    const author_id = '6219ccaefdb5681a29fe4aa5';
    const author_avatar = '/public/images/shejipi.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');
    ;

    try {
      const response = await got({
        method: 'get',
        url: url,
      });

      const $ = cheerio.load(response.data);


      // 下边
      let eles_bottom = $('ul.grid.inf > li');
      for (const el of eles_bottom) {
        let topic = {
          title: $(el).find('h1').text().trim(),
          content: $(el).find('a').attr('href'),
          pic: $(el).find('img').attr('data-src'),
        }
        totalTopics.push(topic);
      }


      for (const topic of totalTopics) {
        topic.original_url = topic.content;
        topic.tab = tab;
        topic.author_id = author_id;
        topic.author_name = author_name;
        topic.author_avatar = author_avatar;
        topic.source = source;
      }

    } catch (e) {
      this.logger.error(e);
    } finally {
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async tianxiazuqiu() {
    let totalTopics = [];

    const tab = 'yutang';
    const author_id = '6294a9b6d32a724c4b39304c';
    const author_name = '天下足球';
    const author_avatar = '/public/images/tianxiazuqiu.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    let url = 'https://tv.cctv.com/lm/txzq/';

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('#boxList > ul > li');
      await this.ctx.helper.autoScrollByLimitSlow(page, 500);
      const eles = await page.$$('#boxList > ul > li');
      for (const el of eles) {
        let title = await el.$eval('.txt a', v => v.innerText);
        let content = await el.$eval('.txt a', v => v.href);
        let pic = await el.$eval('.image img', v => v.src);
        let tags = ['完整版'];
        title = title.replace('[天下足球]', '');
        if (title.includes('豪门恩怨')) {
          tags.push('豪门恩怨');
        }
        let topic = {
          title,
          content,
          original_url: content,
          pic,
          tags,
          tab,
          author_id,
          author_name,
          author_avatar,
          source,
        };
        totalTopics.push(topic);
      }


      await page.click('div.suction > div > div > ul > li:nth-child(2)');
      await page.waitForSelector('#boxList > ul > li');
      await this.ctx.helper.autoScrollByLimitSlow(page, 1000);
      const eles2 = await page.$$('#boxList > ul > li');
      for (const el of eles2) {
        let title = await el.$eval('.txt a', v => v.innerText);
        let content = await el.$eval('.txt a', v => v.href);
        let pic = await el.$eval('.image img', v => v.src);
        let tags = ['精彩片段'];
        title = title.replace('[天下足球]', '');
        if (title.includes('十佳进球')) {
          tags = ['十佳进球'];
        } else if (title.includes('十佳扑救')) {
          tags = ['十佳扑救'];
        }
        if (title.includes('豪门恩怨')) {
          tags.push('豪门恩怨');
        }
        let topic = {
          title,
          content,
          original_url: content,
          pic,
          tags,
          tab,
          author_id,
          author_name,
          author_avatar,
          source,
          is_video: true
        };
        totalTopics.push(topic);
      }

    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async tongcheng() {
    let totalTopics = [];

    // const url = 'http://go.ly.com/travels/home/index.html';
    const url = 'http://www.ly.com/travels/';

    const tab = 'yutang';
    const author_name = '同程旅行';
    const author_id = '620bc1dc8c07a7374f998203';
    const author_avatar = '/public/images/tongcheng.png';
    const source = '同程旅行' + dayjs().format('YYYY-MM-DD');

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});
      await page.click('ul.yjNavList > li:last-child');
      await page.waitForTimeout(7000);
      await page.waitForSelector('ul.yjList > li');
      await this.ctx.helper.autoScrollByLimitSlow(page, 5000);

      const top_eles = await page.$$('.topimgs > .topjb');

      for (const el of top_eles) {
        let title = await el.$eval('.txtp a', v => v.innerText);
        let content = await el.$eval('.txtp a', v => v.href);
        let pic = await el.$eval('a > img', v => v.src);

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          boards: ['旅游'],
          tags: ['旅游'],
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }

      const eles = await page.$$('ul.yjList > li');
      for (const el of eles) {
        const title = await el.$eval('.yjTitle', v => v.innerText);
        let content = await el.$eval('.yjTitle', v => v.getAttribute('data-id'));
        content = `http://www.ly.com/travels/${content}.html`;
        const intro = await el.$eval('.yjInfo', v => v.innerText);
        const pic = await el.$eval('.yjListLeft > img', v => v.src);

        let topic = {
          title,
          content,
          original_url: content,
          intro,
          pic,
          boards: ['旅游'],
          tags: ['旅游'],
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error('同程游记爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async tuniu() {
    const {ctx} = this;

    const agent = new https.Agent({
      rejectUnauthorized: false
    });

    const url = 'https://trips.tuniu.com/travels/index/ajax-list?sortType=3&page=1&limit=10';

    const resp = await axios.get(url, {httpsAgent: agent});

    let items = resp.data.data.rows;

    const source = '途牛网' + dayjs().format('YYYY-MM-DD');
    const totalTopics = [];

    for (const item of items) {
      const topic = {
        title: item.name,
        content: 'https://www.tuniu.com/trips/' + item.id,
        original_url: 'https://www.tuniu.com/trips/' + item.id,
        pic: item.picUrl.replace(/(imageView2.+\/w\/)\d+.+/, '$1500'),
        intro: item.summary,
        visit_count: item.viewCount,
        reply_count: item.commentCount,
        tab: 'yutang',
        boards: ['旅游'],
        tags: ['旅游'],
        author_id: '620bb675bf3180349c03d4f0',
        author_name: '途牛网',
        author_avatar: '/public/images/tuniu.png',
        source,
      }
      totalTopics.push(topic);
    }

    return totalTopics;
  }


  async wuyaocto(url, boards) {
    let totalTopics = [];
    let source = '51CTO' + dayjs().format('YYYY-MM-DD');

    let browser;
    let page;

    try {
      [browser, page] = await this.ctx.helper.getCanUseBrowserAndPage(url, '.article-ir.articleItem');
      let eleHandlers = await page.$$('.article-ir.articleItem');

      for (let i = 0; i < eleHandlers.length; i++) {
        let eleHandler = eleHandlers[i];

        let title = await eleHandler.$eval('.article-irl-ct_title', el => el.innerText);
        let content = await eleHandler.$eval('.article-irl-ct_title', el => el.href);
        let original_url = content;
        let intro = await eleHandler.$eval('.article-abstract', el => el.innerText);

        let tags = await eleHandler.$$eval('.article-irl-cb_sign > a', eles => eles.map(el => el.innerText));

        let tab = 'tech';
        let author_id = '5f743b516865f230256cd8a8';
        let author_name = '51CTO';
        let author_avatar = 'https://taosha01-1253585015.cos.ap-shanghai.myqcloud.com/typora/wuyaocto.jpeg';

        let pic;
        if (await eleHandler.$('.article-irl-img img')) {
          pic = await eleHandler.$eval('.article-irl-img img', el => el.src);
        }

        totalTopics.push({
          title,
          content,
          original_url,
          pic,
          intro,
          boards: [...boards],
          tags,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        });
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      if (browser) {
        await browser.close();
      }
      this.logger.info('51cto爬取成功', boards);
      return totalTopics;
    }
  }

  async yixinli() {
    let totalTopics = [];

    const url = `https://www.xinli001.com/info`;

    const tab = 'yutang';
    const source = '壹心理热榜' + dayjs().format('YYYY-MM-DD');
    const author_name = '壹心理热榜';
    const author_id = '621a2459cdf88326d4df59aa';
    const author_avatar = '/public/images/yixinli.png';

    const browser = await chromium.launch({headless: true});
    try {
      const page = await browser.newPage();
      await page.setExtraHTTPHeaders({'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'});
      await page.goto(url, {waitUntil: 'domcontentloaded'});

      await page.waitForSelector('#articleListM > .item');
      const eles = await page.$$('#articleListM > .item');

      for (const el of eles) {
        if (!await el.$('div.info > a:nth-child(3)')) {
          continue;
        }
        let title = await el.$eval('a.title', v => v.innerText);
        let content = await el.$eval('a.title', v => v.href);
        let intro = await el.$eval('p.desc', v => v.innerText.trim());
        let pic = await el.$eval('img', v => v.src);
        let visit_count = await el.$eval('.statistics', v => v.innerText.replace('阅读', ''));
        visit_count = +visit_count;
        let tag = await el.$eval('div.info > a:nth-child(3)', v => v.innerText.replace('#', ''));
        let tags = [tag];

        let topic = {
          title,
          content,
          original_url: content,
          intro,
          pic,
          visit_count,
          tags,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      await browser.close();
      if (totalTopics.length == 0) {
        this.logger.error('壹心理爬取异常!!!');
      }
      return totalTopics;
    }
  }


  async zaker(url, boards) {
    let totalTopics = [];

    const tab = 'yutang';
    const source = 'ZAKER新闻' + dayjs().format('YYYY-MM-DD');
    const author_name = 'ZAKER新闻';
    const author_id = '6214ff6a97c6ef139f2cded0';
    const author_avatar = '/public/images/zaker.png';

    let browser;
    let page;
    try {
      [browser, page] = await this.ctx.helper.getCanUseBrowserAndPage(url, '.content-block.masonry-brick');
      const eles = await page.$$('.content-block.masonry-brick');
      for (let i = 0; i < eles.length; i++) {
        let el = eles[i];
        let title;
        if (await el.$('.article-title')) {
          title = await el.$eval('.article-title', v => v.innerText);
        } else {
          continue;
        }

        let content = await el.$eval('a', v => v.href);
        let pic;
        if (await el.$('.pic')) {
          pic = await el.$eval('.pic', v => v.outerHTML);
          pic = pic.replace(/.+data-original="(.+jpg)".+/, 'https:$1')
            .replace('https:http://', 'https://');
        }
        let intro;
        if (await el.$('.article-text')) {
          intro = await el.$eval('.article-text', v => v.innerText);
        }
        let visit_count = 0;
        if (await el.$('.article-comment')) {
          visit_count = await el.$eval('.article-comment', v => v.innerText);
          visit_count = +(visit_count.trim().replace(/[^\d]/g, ''));
        }

        let topic = {
          title,
          content,
          original_url: content,
          pic,
          intro,
          visit_count,
          boards,
          tab,
          source,
          author_id,
          author_name,
          author_avatar,
        };

        totalTopics.push(topic);
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      if (browser) {
        await browser.close();
      }
      if (totalTopics.length == 0) {
        this.logger.error(author_name, boards, '爬取异常!!!');
      } else {
        this.logger.info(author_name, boards, 'success...数量为:', totalTopics.length);
      }
      return totalTopics;
    }
  }


  async zhihuDaily() {
    const param = {
      url: 'https://daily.zhihu.com/',
      author_info: {
        author_id: '5f81cba44853fbd293a51f7d',
        author_name: '知乎日报',
        author_avatar: '/public/images/zhihudaily.png'
      },
      topic_info: {
        tab: 'yutang',
        boards: ['百科'],
      },
      selector_info: {
        box: '.wrap',
        title: 'a',
        content: 'a',
        pic: 'img'
      }
    }

    let topics = await this.ctx.service.commonSvc.commonLight(param);
    topics = topics.filter(v => !v.title.includes('如何正确地吐槽'));
    return topics;
  }


  async yutang() {
    let totalTopics = [];

    const url = 'https://mo.fish/';
    const tab = 'yutang';
    const author_name = '鱼塘热榜';
    const author_id = '61d8dbe730f4b3fd96e5ecab';
    const author_avatar = '/public/images/fish.png';
    const source = author_name + dayjs().format('YYYY-MM-DD');

    const browser = await chromium.launch({headless: true});

    try {
      const context = await browser.newContext();
      const page = await context.newPage();
      await page.goto('https://mo.fish/', {waitUntil: 'domcontentloaded', timeout: 10000});
      await page.waitForSelector('#web-article-list > .list-item', {timeout: 10000});

      let eles = await page.$$('#web-article-list > .list-item .list-item-title');
      let cnts = eles.length;

      this.logger.info('鱼塘本次捞到的数量', cnts);

      while (cnts < 30) {
        await page.waitForTimeout(1000);
        // await page.reload();
        await page.goto('https://mo.fish/', {waitUntil: 'domcontentloaded', timeout: 10000});
        await page.waitForSelector('#web-article-list > .list-item', {timeout: 10000});
        eles = await page.$$('#web-article-list > .list-item .list-item-title');
        cnts = eles.length;
        this.logger.info('鱼塘本次捞到的数量', cnts);
      }

      for (let i = 0; i < cnts; i++) {
        let newPage;
        try {
          let idx = `#web-article-list > .list-item:nth-of-type(${i + 1}) .list-item-title`;
          await page.waitForSelector('#web-article-list > .list-item', {timeout: 10000, state: 'attached'});
          [newPage] = await Promise.all([
            context.waitForEvent('page', {timeout: 10000}),
            page.click(idx),
          ]);


          const title = await page.$eval(idx, el => el.innerText);
          let original_url = newPage.url();

          if (original_url.startsWith('https://passport.weibo.com')) {
            topic.not_import = true;
            // original_url = decodeURIComponent(original_url);
            // original_url = original_url
            //   .replace(/.+(https:\/\/s.weibo.com.+&Refer=top).+/, '$1')
            //   .replace(/.+(https:\/\/s.weibo.com.+)&domain.+/, '$1')
            //   .replace(/.+(https:\/\/weibo.com.+)\?from=.+/, '$1')
            //   .replace('/tv/tv/', '/tv/');
          } else if (original_url.includes('autohome.com.cn')) {
            original_url = decodeURIComponent(original_url).replace(/http:\/\/.+backurl=/, 'https:');
          } else if (original_url.includes('https://www.toutiao.com//toutiao.com')) {
            original_url = original_url.replace('https://www.toutiao.com//', 'https://');
          } else if (original_url.startsWith('https://mp.weixin.qq.com')) {
            if (await newPage.$('.weui-msg__title.warn')) {
              this.logger.info(i, cnts, '微信已删除文章...');
              topic.not_import = true;
            } else if (await newPage.$('.weui-icon-warn')) {
              this.logger.info(i, cnts, '微信链接已过期文章...');
              topic.not_import = true;
            }
          } else if (original_url.startsWith('https://mo.fish')) {
            topic.not_import = true;
          }

          let topic = {};
          topic.title = title;
          topic.content = original_url;
          topic.original_url = original_url;
          topic.tab = tab;
          topic.source = source;
          topic.author_id = author_id;
          topic.author_name = author_name;
          topic.author_avatar = author_avatar;

          console.log(i, cnts, topic.content);
          totalTopics.push(topic);
        } catch (e) {
          console.error(e.message);
        } finally {
          if (newPage) {
            newPage.close();
          }
        }
      }
    } catch (e) {
      this.logger.error(e);
    } finally {
      browser.close();
      if (totalTopics.length == 0) {
        this.logger.error(author_name, '爬取异常!!!');
      }
      const result = this.yutangPostProcessing(totalTopics);

      result.forEach(t => {
        if (t.author_name == '知乎推荐') {
          t.boards = ['传媒'];
          t.tags = ['传媒'];
          if (t.content.includes('zhihu.com/zvideo')) {
            t.boards.push('视频');
            t.tags.push('视频');
          }
        }
      });

      return result;
    }
  }


  yutangPostProcessing(totalTopics) {
    this.ctx.helper.setRealAuthorName(totalTopics);
    const yutangIgnoreUrl = [
      '.xilu.com',
      'kuaixun.stcn.com',
      '.huanqiu.com',
      '.chinanews.com.cn',
      '.myzaker.com',
      '.thepaper.cn',
      '.guancha.cn',
      '.huxiu.com',
      '.jfdaily.com',
      '.jiemian.com',
      'new.qq.com',
      '.kankanews.com',
      '.sohu.com',
      'www.baidu.com',
      'zhihu.com',
      'weibo.com',
      'weibo.cn',
      '.easeus.com',
      '.haiwainet.cn',
      '360doc.com',
      'hostloc.com',
      'time-weekly.com',
      'qbitai.com',
      'sspai.com',
      '163.com',
      '.mysmth.net',
      '.chouti.com',
      'wanzhou.cbg.cn',
      'forgifs.com',
      '.taiwan.cn',
      '.21jingji.com',
      'hebei.com.cn',
      'bbs.hupu.com',
      'www.flyert.com',
      'taobao.com',
      'www.acfun.cn',
      'bbs.tianya.cn',
      'bbs.nga.cn',
      'just998.com',
      '52pojie.cn',
      'nb.zol.com',
      'keylol.com',
      'gamersky.com',
      'kdslife.com',
      '3dmgame.com',
      'lapin.ithome.com',
      'xueqiu.com',
      'jinse.com',
      'sinaimg.cn',
      'api.tophub.fun',
      'github.com',
      'sports.qq.com',
      'caixin.com',
      'ali213.net',
      '17173.com',
      '178.com',
      'ex.chinadaily.com.cn',
      'saraba1st.com',
      '8btc.com',
      'bjnews.com',
      'uuu9.com',
      'weread.qq.com',
      'zhidao.baidu.com/day',
      'ld246.com',
      'mafengwo.cn',
      'page.om.qq.com',
      'chiphell.com',
      'guokr.com',
      'jandan.net',
      'studygolang.com',
      'cnhubei.com',
      'eastday.com',
      'parallels.cn',
      'gouhuo.qq.com',
      'account.ltaaa.cn',
      'laosiji.com',
      'jiandan.net',
      'learnku.com',
      'msn.cn',
      '81.cn',
      'flyertea.com',
      'view.inews.qq.com',
      'geekhub.com',
      'huoxing24.com',
      'kepuchina.cn',
    ];

    let result = [];
    for (let i = 0; i < totalTopics.length; i++) {
      let topic = totalTopics[i];
      topic.visit_count = Math.round(Math.random() * (200 - 10)) + 10;
      if (topic.title.includes('鱼塘热榜') && topic.title.includes('专')) {
        topic.not_import = true;
      } else if (topic.title.includes('王者荣耀')) {
        topic.not_import = true;
      } else if (topic.title.includes('百货 50 条')) {
        topic.not_import = true;
      } else if (/\[博海拾贝\d{4}/.test(topic.title)) {
        topic.not_import = true;
      }

      if (topic.title.startsWith('【')) {
        topic.title = topic.title.substr(1, topic.title.indexOf('】') - 1);
      }
      if (topic.title.includes('\n')) {
        topic.title = topic.title.split('\n')[0];
      }

      if (topic.title.startsWith('@')) {
        topic.not_import = true;
      }

      if (topic.title.length > 70) {
        topic.not_import = true;
      }

      if (topic.title.includes('每日好价')) {
        topic.not_import = true;
      }

      if (!topic.content) {
        topic.not_import = true;
      }

      if (topic.content.startsWith('chrome-error')) {
        this.logger.warn(`${i}, ${totalTopics.length}, chrome-error`);
        topic.not_import = true;
      }

      yutangIgnoreUrl.forEach(v => {
        if (topic.content.includes(v)) {
          topic.not_import = true;
        }
      });

      if (!topic.not_import) {
        result.push(topic);
      }
    }
    return result;
  }


  async getXyzPicAndIntro(url) {
    const response = await got(url);
    const $ = cheerio.load(response.data);
    const result = [
      $('.avatar-wrapper > img').attr('src'),
      $('#__next > div.jsx-3092373605.jsx-620427904 > main > main > div.jsx-f56e5c7f5d0d4b11.tabs-container > div > div').text().replace(/\n/g, '').replace(/\s{2,}/g, ' ').trim(),
      $('#__next > div > main > div.wrap > div.title').text(),
    ];
    return result;
  }


  async getXimalayaUserameAndPicAndIntro(url) {
    const response = await got(url);
    const $ = cheerio.load(response.data);
    const result = [
      $('h1.title._qj').text(),
      'https:' + $('img.img._qj').attr('src'),
      $('article.intro.Q_v').text(),
    ];
    return result;
  }


  async getXyzTopicTimes(url) {
    let result = [];
    const response = await got(url);
    const $ = cheerio.load(response.data);
    const elements = $('ul.tab > li').get();
    for (let i = 0; i < elements.length; i++) {
      const item = elements[i];
      const title = $(item).find('.title').first().text().trim();
      const time = $(item).find('time').attr('datetime');
      result.push({title, time});
    }

    return result;
  }

}

module.exports = CrawlerSvc;
